lsst.pipe.tasks  21.0.0-132-gbeff0d17+eb6a08541c
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import logging
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.geom as geom
30 import lsst.afw.geom as afwGeom
31 import lsst.afw.image as afwImage
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 import lsst.afw.detection as afwDet
35 import lsst.coadd.utils as coaddUtils
36 import lsst.pipe.base as pipeBase
37 import lsst.meas.algorithms as measAlg
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from .healSparseMapping import HealSparseInputMapTask
48 from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack
49 from lsst.daf.butler import DeferredDatasetHandle
50 
51 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
52  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
53  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
54 
55 log = logging.getLogger(__name__.partition(".")[2])
56 
57 
58 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
59  dimensions=("tract", "patch", "band", "skymap"),
60  defaultTemplates={"inputCoaddName": "deep",
61  "outputCoaddName": "deep",
62  "warpType": "direct",
63  "warpTypeSuffix": ""}):
64 
65  inputWarps = pipeBase.connectionTypes.Input(
66  doc=("Input list of warps to be assemebled i.e. stacked."
67  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
68  name="{inputCoaddName}Coadd_{warpType}Warp",
69  storageClass="ExposureF",
70  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
71  deferLoad=True,
72  multiple=True
73  )
74  skyMap = pipeBase.connectionTypes.Input(
75  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
76  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
77  storageClass="SkyMap",
78  dimensions=("skymap", ),
79  )
80  selectedVisits = pipeBase.connectionTypes.Input(
81  doc="Selected visits to be coadded.",
82  name="{outputCoaddName}Visits",
83  storageClass="StructuredDataDict",
84  dimensions=("instrument", "tract", "patch", "skymap", "band")
85  )
86  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
87  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
88  " BRIGHT_OBJECT."),
89  name="brightObjectMask",
90  storageClass="ObjectMaskCatalog",
91  dimensions=("tract", "patch", "skymap", "band"),
92  )
93  coaddExposure = pipeBase.connectionTypes.Output(
94  doc="Output coadded exposure, produced by stacking input warps",
95  name="{outputCoaddName}Coadd{warpTypeSuffix}",
96  storageClass="ExposureF",
97  dimensions=("tract", "patch", "skymap", "band"),
98  )
99  nImage = pipeBase.connectionTypes.Output(
100  doc="Output image of number of input images per pixel",
101  name="{outputCoaddName}Coadd_nImage",
102  storageClass="ImageU",
103  dimensions=("tract", "patch", "skymap", "band"),
104  )
105  inputMap = pipeBase.connectionTypes.Output(
106  doc="Output healsparse map of input images",
107  name="{outputCoaddName}Coadd_inputMap",
108  storageClass="HealSparseMap",
109  dimensions=("tract", "patch", "skymap", "band"),
110  )
111 
112  def __init__(self, *, config=None):
113  super().__init__(config=config)
114 
115  # Override the connection's name template with config to replicate Gen2 behavior
116  # This duplicates some of the logic in the base class, due to wanting Gen2 and
117  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
118  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
119  templateValues['warpType'] = config.warpType
120  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
121  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
122  for name in self.allConnections}
123  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
124  # End code to remove after deprecation
125 
126  if not config.doMaskBrightObjects:
127  self.prerequisiteInputs.remove("brightObjectMask")
128 
129  if not config.doSelectVisits:
130  self.inputs.remove("selectedVisits")
131 
132  if not config.doNImage:
133  self.outputs.remove("nImage")
134 
135  if not self.config.doInputMap:
136  self.outputs.remove("inputMap")
137 
138 
139 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
140  pipelineConnections=AssembleCoaddConnections):
141  """Configuration parameters for the `AssembleCoaddTask`.
142 
143  Notes
144  -----
145  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
146  only set the bitplane config.brightObjectMaskName. To make this useful you
147  *must* also configure the flags.pixel algorithm, for example by adding
148 
149  .. code-block:: none
150 
151  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
152  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
153 
154  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
155  """
156  warpType = pexConfig.Field(
157  doc="Warp name: one of 'direct' or 'psfMatched'",
158  dtype=str,
159  default="direct",
160  )
161  subregionSize = pexConfig.ListField(
162  dtype=int,
163  doc="Width, height of stack subregion size; "
164  "make small enough that a full stack of images will fit into memory at once.",
165  length=2,
166  default=(2000, 2000),
167  )
168  statistic = pexConfig.Field(
169  dtype=str,
170  doc="Main stacking statistic for aggregating over the epochs.",
171  default="MEANCLIP",
172  )
173  doOnlineForMean = pexConfig.Field(
174  dtype=bool,
175  doc="Perform online coaddition when statistic=\"MEAN\" to save memory?",
176  default=False,
177  )
178  doSigmaClip = pexConfig.Field(
179  dtype=bool,
180  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
181  default=False,
182  )
183  sigmaClip = pexConfig.Field(
184  dtype=float,
185  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
186  default=3.0,
187  )
188  clipIter = pexConfig.Field(
189  dtype=int,
190  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
191  default=2,
192  )
193  calcErrorFromInputVariance = pexConfig.Field(
194  dtype=bool,
195  doc="Calculate coadd variance from input variance by stacking statistic."
196  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
197  default=True,
198  )
199  scaleZeroPoint = pexConfig.ConfigurableField(
200  target=ScaleZeroPointTask,
201  doc="Task to adjust the photometric zero point of the coadd temp exposures",
202  )
203  doInterp = pexConfig.Field(
204  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
205  dtype=bool,
206  default=True,
207  )
208  interpImage = pexConfig.ConfigurableField(
209  target=InterpImageTask,
210  doc="Task to interpolate (and extrapolate) over NaN pixels",
211  )
212  doWrite = pexConfig.Field(
213  doc="Persist coadd?",
214  dtype=bool,
215  default=True,
216  )
217  doNImage = pexConfig.Field(
218  doc="Create image of number of contributing exposures for each pixel",
219  dtype=bool,
220  default=False,
221  )
222  doUsePsfMatchedPolygons = pexConfig.Field(
223  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
224  dtype=bool,
225  default=False,
226  )
227  maskPropagationThresholds = pexConfig.DictField(
228  keytype=str,
229  itemtype=float,
230  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
231  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
232  "would have contributed exceeds this value."),
233  default={"SAT": 0.1},
234  )
235  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
236  doc="Mask planes to remove before coadding")
237  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
238  doc="Set mask and flag bits for bright objects?")
239  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
240  doc="Name of mask bit used for bright objects")
241  coaddPsf = pexConfig.ConfigField(
242  doc="Configuration for CoaddPsf",
243  dtype=measAlg.CoaddPsfConfig,
244  )
245  doAttachTransmissionCurve = pexConfig.Field(
246  dtype=bool, default=False, optional=False,
247  doc=("Attach a piecewise TransmissionCurve for the coadd? "
248  "(requires all input Exposures to have TransmissionCurves).")
249  )
250  hasFakes = pexConfig.Field(
251  dtype=bool,
252  default=False,
253  doc="Should be set to True if fake sources have been inserted into the input data."
254  )
255  doSelectVisits = pexConfig.Field(
256  doc="Coadd only visits selected by a SelectVisitsTask",
257  dtype=bool,
258  default=False,
259  )
260  doInputMap = pexConfig.Field(
261  doc="Create a bitwise map of coadd inputs",
262  dtype=bool,
263  default=False,
264  )
265  inputMapper = pexConfig.ConfigurableField(
266  doc="Input map creation subtask.",
267  target=HealSparseInputMapTask,
268  )
269 
270  def setDefaults(self):
271  super().setDefaults()
272  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
273 
274  def validate(self):
275  super().validate()
276  if self.doPsfMatch:
277  # Backwards compatibility.
278  # Configs do not have loggers
279  log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
280  self.warpType = 'psfMatched'
281  if self.doSigmaClip and self.statistic != "MEANCLIP":
282  log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
283  self.statistic = "MEANCLIP"
284  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
285  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
286  "compute and set a non-zero coadd variance estimate." % (self.statistic))
287 
288  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
289  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
290  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
291  if str(k) not in unstackableStats]
292  raise ValueError("statistic %s is not allowed. Please choose one of %s."
293  % (self.statistic, stackableStats))
294 
295 
296 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
297  """Assemble a coadded image from a set of warps (coadded temporary exposures).
298 
299  We want to assemble a coadded image from a set of Warps (also called
300  coadded temporary exposures or ``coaddTempExps``).
301  Each input Warp covers a patch on the sky and corresponds to a single
302  run/visit/exposure of the covered patch. We provide the task with a list
303  of Warps (``selectDataList``) from which it selects Warps that cover the
304  specified patch (pointed at by ``dataRef``).
305  Each Warp that goes into a coadd will typically have an independent
306  photometric zero-point. Therefore, we must scale each Warp to set it to
307  a common photometric zeropoint. WarpType may be one of 'direct' or
308  'psfMatched', and the boolean configs `config.makeDirect` and
309  `config.makePsfMatched` set which of the warp types will be coadded.
310  The coadd is computed as a mean with optional outlier rejection.
311  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
312  Finally, Warps can have bad 'NaN' pixels which received no input from the
313  source calExps. We interpolate over these bad (NaN) pixels.
314 
315  `AssembleCoaddTask` uses several sub-tasks. These are
316 
317  - `ScaleZeroPointTask`
318  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
319  - `InterpImageTask`
320  - interpolate across bad pixels (NaN) in the final coadd
321 
322  You can retarget these subtasks if you wish.
323 
324  Notes
325  -----
326  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
327  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
328  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
329  no debug variables of its own. Some of the subtasks may support debug
330  variables. See the documentation for the subtasks for further information.
331 
332  Examples
333  --------
334  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
335  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
336  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
337  inputs: a data reference to the tract patch and filter to be coadded, and
338  a list of Warps to attempt to coadd. These are specified using ``--id`` and
339  ``--selectId``, respectively:
340 
341  .. code-block:: none
342 
343  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
344  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
345 
346  Only the Warps that cover the specified tract and patch will be coadded.
347  A list of the available optional arguments can be obtained by calling
348  ``assembleCoadd.py`` with the ``--help`` command line argument:
349 
350  .. code-block:: none
351 
352  assembleCoadd.py --help
353 
354  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
355  multi-band processing, we will generate the HSC-I & -R band coadds from
356  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
357  assuming that the lsst stack has been already set up, we must set up the
358  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
359  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
360  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
361  coadds, we must first
362 
363  - processCcd
364  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
365  - makeSkyMap
366  - create a skymap that covers the area of the sky present in the raw exposures
367  - makeCoaddTempExp
368  - warp the individual calibrated exposures to the tangent plane of the coadd
369 
370  We can perform all of these steps by running
371 
372  .. code-block:: none
373 
374  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
375 
376  This will produce warped exposures for each visit. To coadd the warped
377  data, we call assembleCoadd.py as follows:
378 
379  .. code-block:: none
380 
381  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
382  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
383  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
384  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
385  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
386  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
387  --selectId visit=903988 ccd=24
388 
389  that will process the HSC-I band data. The results are written in
390  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
391 
392  You may also choose to run:
393 
394  .. code-block:: none
395 
396  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
397  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
398  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
399  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
400  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
401  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
402  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
403  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
404 
405  to generate the coadd for the HSC-R band if you are interested in
406  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
407  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
408  rather than `AssembleCoaddTask` to make the coadd.
409  """
410  ConfigClass = AssembleCoaddConfig
411  _DefaultName = "assembleCoadd"
412 
413  def __init__(self, *args, **kwargs):
414  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
415  if args:
416  argNames = ["config", "name", "parentTask", "log"]
417  kwargs.update({k: v for k, v in zip(argNames, args)})
418  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
419  "PipelineTask will not take positional args" % argNames, FutureWarning)
420 
421  super().__init__(**kwargs)
422  self.makeSubtask("interpImage")
423  self.makeSubtask("scaleZeroPoint")
424 
425  if self.config.doMaskBrightObjects:
426  mask = afwImage.Mask()
427  try:
428  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
429  except pexExceptions.LsstCppException:
430  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
431  mask.getMaskPlaneDict().keys())
432  del mask
433 
434  if self.config.doInputMap:
435  self.makeSubtask("inputMapper")
436 
437  self.warpType = self.config.warpType
438 
439  @utils.inheritDoc(pipeBase.PipelineTask)
440  def runQuantum(self, butlerQC, inputRefs, outputRefs):
441  # Docstring to be formatted with info from PipelineTask.runQuantum
442  """
443  Notes
444  -----
445  Assemble a coadd from a set of Warps.
446 
447  PipelineTask (Gen3) entry point to Coadd a set of Warps.
448  Analogous to `runDataRef`, it prepares all the data products to be
449  passed to `run`, and processes the results before returning a struct
450  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
451  Therefore, its inputs are accessed subregion by subregion
452  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
453  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
454  correspond to an update in `runDataRef` while both entry points
455  are used.
456  """
457  inputData = butlerQC.get(inputRefs)
458 
459  # Construct skyInfo expected by run
460  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
461  skyMap = inputData["skyMap"]
462  outputDataId = butlerQC.quantum.dataId
463 
464  inputData['skyInfo'] = makeSkyInfo(skyMap,
465  tractId=outputDataId['tract'],
466  patchId=outputDataId['patch'])
467 
468  if self.config.doSelectVisits:
469  warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
470  else:
471  warpRefList = inputData['inputWarps']
472 
473  # Perform same middle steps as `runDataRef` does
474  inputs = self.prepareInputs(warpRefList)
475  self.log.info("Found %d %s", len(inputs.tempExpRefList),
476  self.getTempExpDatasetName(self.warpType))
477  if len(inputs.tempExpRefList) == 0:
478  raise pipeBase.NoWorkFound("No coadd temporary exposures found")
479 
480  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
481  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
482  inputs.weightList, supplementaryData=supplementaryData)
483 
484  inputData.setdefault('brightObjectMask', None)
485  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
486 
487  if self.config.doWrite:
488  butlerQC.put(retStruct, outputRefs)
489  return retStruct
490 
491  @pipeBase.timeMethod
492  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
493  """Assemble a coadd from a set of Warps.
494 
495  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
496  Compute weights to be applied to each Warp and
497  find scalings to match the photometric zeropoint to a reference Warp.
498  Assemble the Warps using `run`. Interpolate over NaNs and
499  optionally write the coadd to disk. Return the coadded exposure.
500 
501  Parameters
502  ----------
503  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
504  Data reference defining the patch for coaddition and the
505  reference Warp (if ``config.autoReference=False``).
506  Used to access the following data products:
507  - ``self.config.coaddName + "Coadd_skyMap"``
508  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
509  - ``self.config.coaddName + "Coadd"``
510  selectDataList : `list`
511  List of data references to Calexps. Data to be coadded will be
512  selected from this list based on overlap with the patch defined
513  by dataRef, grouped by visit, and converted to a list of data
514  references to warps.
515  warpRefList : `list`
516  List of data references to Warps to be coadded.
517  Note: `warpRefList` is just the new name for `tempExpRefList`.
518 
519  Returns
520  -------
521  retStruct : `lsst.pipe.base.Struct`
522  Result struct with components:
523 
524  - ``coaddExposure``: coadded exposure (``Exposure``).
525  - ``nImage``: exposure count image (``Image``).
526  """
527  if selectDataList and warpRefList:
528  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
529  "and which to use is ambiguous. Please pass only one.")
530 
531  skyInfo = self.getSkyInfo(dataRef)
532  if warpRefList is None:
533  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
534  if len(calExpRefList) == 0:
535  self.log.warning("No exposures to coadd")
536  return
537  self.log.info("Coadding %d exposures", len(calExpRefList))
538 
539  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
540 
541  inputData = self.prepareInputs(warpRefList)
542  self.log.info("Found %d %s", len(inputData.tempExpRefList),
543  self.getTempExpDatasetName(self.warpType))
544  if len(inputData.tempExpRefList) == 0:
545  self.log.warning("No coadd temporary exposures found")
546  return
547 
548  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
549 
550  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
551  inputData.weightList, supplementaryData=supplementaryData)
552 
553  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
554  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
555 
556  if self.config.doWrite:
557  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
558  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
559  else:
560  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
561  self.log.info("Persisting %s", coaddDatasetName)
562  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
563  if self.config.doNImage and retStruct.nImage is not None:
564  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
565 
566  return retStruct
567 
568  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
569  """Interpolate over missing data and mask bright stars.
570 
571  Parameters
572  ----------
573  coaddExposure : `lsst.afw.image.Exposure`
574  The coadded exposure to process.
575  dataRef : `lsst.daf.persistence.ButlerDataRef`
576  Butler data reference for supplementary data.
577  """
578  if self.config.doInterp:
579  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
580  # The variance must be positive; work around for DM-3201.
581  varArray = coaddExposure.variance.array
582  with numpy.errstate(invalid="ignore"):
583  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
584 
585  if self.config.doMaskBrightObjects:
586  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
587 
588  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
589  """Make additional inputs to run() specific to subclasses (Gen2)
590 
591  Duplicates interface of `runDataRef` method
592  Available to be implemented by subclasses only if they need the
593  coadd dataRef for performing preliminary processing before
594  assembling the coadd.
595 
596  Parameters
597  ----------
598  dataRef : `lsst.daf.persistence.ButlerDataRef`
599  Butler data reference for supplementary data.
600  selectDataList : `list` (optional)
601  Optional List of data references to Calexps.
602  warpRefList : `list` (optional)
603  Optional List of data references to Warps.
604  """
605  return pipeBase.Struct()
606 
607  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
608  """Make additional inputs to run() specific to subclasses (Gen3)
609 
610  Duplicates interface of `runQuantum` method.
611  Available to be implemented by subclasses only if they need the
612  coadd dataRef for performing preliminary processing before
613  assembling the coadd.
614 
615  Parameters
616  ----------
617  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
618  Gen3 Butler object for fetching additional data products before
619  running the Task specialized for quantum being processed
620  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
621  Attributes are the names of the connections describing input dataset types.
622  Values are DatasetRefs that task consumes for corresponding dataset type.
623  DataIds are guaranteed to match data objects in ``inputData``.
624  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
625  Attributes are the names of the connections describing output dataset types.
626  Values are DatasetRefs that task is to produce
627  for corresponding dataset type.
628  """
629  return pipeBase.Struct()
630 
631  def getTempExpRefList(self, patchRef, calExpRefList):
632  """Generate list data references corresponding to warped exposures
633  that lie within the patch to be coadded.
634 
635  Parameters
636  ----------
637  patchRef : `dataRef`
638  Data reference for patch.
639  calExpRefList : `list`
640  List of data references for input calexps.
641 
642  Returns
643  -------
644  tempExpRefList : `list`
645  List of Warp/CoaddTempExp data references.
646  """
647  butler = patchRef.getButler()
648  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
649  self.getTempExpDatasetName(self.warpType))
650  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
651  g, groupData.keys) for
652  g in groupData.groups.keys()]
653  return tempExpRefList
654 
655  def prepareInputs(self, refList):
656  """Prepare the input warps for coaddition by measuring the weight for
657  each warp and the scaling for the photometric zero point.
658 
659  Each Warp has its own photometric zeropoint and background variance.
660  Before coadding these Warps together, compute a scale factor to
661  normalize the photometric zeropoint and compute the weight for each Warp.
662 
663  Parameters
664  ----------
665  refList : `list`
666  List of data references to tempExp
667 
668  Returns
669  -------
670  result : `lsst.pipe.base.Struct`
671  Result struct with components:
672 
673  - ``tempExprefList``: `list` of data references to tempExp.
674  - ``weightList``: `list` of weightings.
675  - ``imageScalerList``: `list` of image scalers.
676  """
677  statsCtrl = afwMath.StatisticsControl()
678  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
679  statsCtrl.setNumIter(self.config.clipIter)
680  statsCtrl.setAndMask(self.getBadPixelMask())
681  statsCtrl.setNanSafe(True)
682  # compute tempExpRefList: a list of tempExpRef that actually exist
683  # and weightList: a list of the weight of the associated coadd tempExp
684  # and imageScalerList: a list of scale factors for the associated coadd tempExp
685  tempExpRefList = []
686  weightList = []
687  imageScalerList = []
688  tempExpName = self.getTempExpDatasetName(self.warpType)
689  for tempExpRef in refList:
690  # Gen3's DeferredDatasetHandles are guaranteed to exist and
691  # therefore have no datasetExists() method
692  if not isinstance(tempExpRef, DeferredDatasetHandle):
693  if not tempExpRef.datasetExists(tempExpName):
694  self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
695  continue
696 
697  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
698  # Ignore any input warp that is empty of data
699  if numpy.isnan(tempExp.image.array).all():
700  continue
701  maskedImage = tempExp.getMaskedImage()
702  imageScaler = self.scaleZeroPoint.computeImageScaler(
703  exposure=tempExp,
704  dataRef=tempExpRef,
705  )
706  try:
707  imageScaler.scaleMaskedImage(maskedImage)
708  except Exception as e:
709  self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
710  continue
711  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
712  afwMath.MEANCLIP, statsCtrl)
713  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
714  weight = 1.0 / float(meanVar)
715  if not numpy.isfinite(weight):
716  self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId)
717  continue
718  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
719 
720  del maskedImage
721  del tempExp
722 
723  tempExpRefList.append(tempExpRef)
724  weightList.append(weight)
725  imageScalerList.append(imageScaler)
726 
727  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
728  imageScalerList=imageScalerList)
729 
730  def prepareStats(self, mask=None):
731  """Prepare the statistics for coadding images.
732 
733  Parameters
734  ----------
735  mask : `int`, optional
736  Bit mask value to exclude from coaddition.
737 
738  Returns
739  -------
740  stats : `lsst.pipe.base.Struct`
741  Statistics structure with the following fields:
742 
743  - ``statsCtrl``: Statistics control object for coadd
744  (`lsst.afw.math.StatisticsControl`)
745  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
746  """
747  if mask is None:
748  mask = self.getBadPixelMask()
749  statsCtrl = afwMath.StatisticsControl()
750  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
751  statsCtrl.setNumIter(self.config.clipIter)
752  statsCtrl.setAndMask(mask)
753  statsCtrl.setNanSafe(True)
754  statsCtrl.setWeighted(True)
755  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
756  for plane, threshold in self.config.maskPropagationThresholds.items():
757  bit = afwImage.Mask.getMaskPlane(plane)
758  statsCtrl.setMaskPropagationThreshold(bit, threshold)
759  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
760  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
761 
762  @pipeBase.timeMethod
763  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
764  altMaskList=None, mask=None, supplementaryData=None):
765  """Assemble a coadd from input warps
766 
767  Assemble the coadd using the provided list of coaddTempExps. Since
768  the full coadd covers a patch (a large area), the assembly is
769  performed over small areas on the image at a time in order to
770  conserve memory usage. Iterate over subregions within the outer
771  bbox of the patch using `assembleSubregion` to stack the corresponding
772  subregions from the coaddTempExps with the statistic specified.
773  Set the edge bits the coadd mask based on the weight map.
774 
775  Parameters
776  ----------
777  skyInfo : `lsst.pipe.base.Struct`
778  Struct with geometric information about the patch.
779  tempExpRefList : `list`
780  List of data references to Warps (previously called CoaddTempExps).
781  imageScalerList : `list`
782  List of image scalers.
783  weightList : `list`
784  List of weights
785  altMaskList : `list`, optional
786  List of alternate masks to use rather than those stored with
787  tempExp.
788  mask : `int`, optional
789  Bit mask value to exclude from coaddition.
790  supplementaryData : lsst.pipe.base.Struct, optional
791  Struct with additional data products needed to assemble coadd.
792  Only used by subclasses that implement `makeSupplementaryData`
793  and override `run`.
794 
795  Returns
796  -------
797  result : `lsst.pipe.base.Struct`
798  Result struct with components:
799 
800  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
801  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
802  - ``inputMap``: bit-wise map of inputs, if requested.
803  - ``warpRefList``: input list of refs to the warps (
804  ``lsst.daf.butler.DeferredDatasetHandle`` or
805  ``lsst.daf.persistence.ButlerDataRef``)
806  (unmodified)
807  - ``imageScalerList``: input list of image scalers (unmodified)
808  - ``weightList``: input list of weights (unmodified)
809  """
810  tempExpName = self.getTempExpDatasetName(self.warpType)
811  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
812  stats = self.prepareStats(mask=mask)
813 
814  if altMaskList is None:
815  altMaskList = [None]*len(tempExpRefList)
816 
817  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
818  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
819  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
820  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
821  coaddMaskedImage = coaddExposure.getMaskedImage()
822  subregionSizeArr = self.config.subregionSize
823  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
824  # if nImage is requested, create a zero one which can be passed to assembleSubregion
825  if self.config.doNImage:
826  nImage = afwImage.ImageU(skyInfo.bbox)
827  else:
828  nImage = None
829  # If inputMap is requested, create the initial version that can be masked in
830  # assembleSubregion.
831  if self.config.doInputMap:
832  self.inputMapper.build_ccd_input_map(skyInfo.bbox,
833  skyInfo.wcs,
834  coaddExposure.getInfo().getCoaddInputs().ccds)
835 
836  if self.config.doOnlineForMean and self.config.statistic == "MEAN":
837  try:
838  self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
839  weightList, altMaskList, stats.ctrl,
840  nImage=nImage)
841  except Exception as e:
842  self.log.fatal("Cannot compute online coadd %s", e)
843  else:
844  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
845  try:
846  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
847  weightList, altMaskList, stats.flags, stats.ctrl,
848  nImage=nImage)
849  except Exception as e:
850  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
851 
852  # If inputMap is requested, we must finalize the map after the accumulation.
853  if self.config.doInputMap:
854  self.inputMapper.finalize_ccd_input_map_mask()
855  inputMap = self.inputMapper.ccd_input_map
856  else:
857  inputMap = None
858 
859  self.setInexactPsf(coaddMaskedImage.getMask())
860  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
861  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
862  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
863  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
864  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
865  weightList=weightList, inputMap=inputMap)
866 
867  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
868  """Set the metadata for the coadd.
869 
870  This basic implementation sets the filter from the first input.
871 
872  Parameters
873  ----------
874  coaddExposure : `lsst.afw.image.Exposure`
875  The target exposure for the coadd.
876  tempExpRefList : `list`
877  List of data references to tempExp.
878  weightList : `list`
879  List of weights.
880  """
881  assert len(tempExpRefList) == len(weightList), "Length mismatch"
882  tempExpName = self.getTempExpDatasetName(self.warpType)
883  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
884  # (and we need more than just the PropertySet that contains the header), which is not possible
885  # with the current butler (see #2777).
886  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
887 
888  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
889  # Gen 3 API
890  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
891  else:
892  # Gen 2 API. Delete this when Gen 2 retired
893  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
894  for tempExpRef in tempExpRefList]
895  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
896 
897  # Set the coadd FilterLabel to the band of the first input exposure:
898  # Coadds are calibrated, so the physical label is now meaningless.
899  coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
900  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
901  coaddInputs.ccds.reserve(numCcds)
902  coaddInputs.visits.reserve(len(tempExpList))
903 
904  for tempExp, weight in zip(tempExpList, weightList):
905  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
906 
907  if self.config.doUsePsfMatchedPolygons:
908  self.shrinkValidPolygons(coaddInputs)
909 
910  coaddInputs.visits.sort()
911  if self.warpType == "psfMatched":
912  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
913  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
914  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
915  # having the maximum width (sufficient because square)
916  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
917  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
918  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
919  else:
920  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
921  self.config.coaddPsf.makeControl())
922  coaddExposure.setPsf(psf)
923  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
924  coaddExposure.getWcs())
925  coaddExposure.getInfo().setApCorrMap(apCorrMap)
926  if self.config.doAttachTransmissionCurve:
927  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
928  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
929 
930  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
931  altMaskList, statsFlags, statsCtrl, nImage=None):
932  """Assemble the coadd for a sub-region.
933 
934  For each coaddTempExp, check for (and swap in) an alternative mask
935  if one is passed. Remove mask planes listed in
936  `config.removeMaskPlanes`. Finally, stack the actual exposures using
937  `lsst.afw.math.statisticsStack` with the statistic specified by
938  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
939  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
940  an N-sigma clipped mean where N and iterations are specified by
941  statsCtrl. Assign the stacked subregion back to the coadd.
942 
943  Parameters
944  ----------
945  coaddExposure : `lsst.afw.image.Exposure`
946  The target exposure for the coadd.
947  bbox : `lsst.geom.Box`
948  Sub-region to coadd.
949  tempExpRefList : `list`
950  List of data reference to tempExp.
951  imageScalerList : `list`
952  List of image scalers.
953  weightList : `list`
954  List of weights.
955  altMaskList : `list`
956  List of alternate masks to use rather than those stored with
957  tempExp, or None. Each element is dict with keys = mask plane
958  name to which to add the spans.
959  statsFlags : `lsst.afw.math.Property`
960  Property object for statistic for coadd.
961  statsCtrl : `lsst.afw.math.StatisticsControl`
962  Statistics control object for coadd.
963  nImage : `lsst.afw.image.ImageU`, optional
964  Keeps track of exposure count for each pixel.
965  """
966  self.log.debug("Computing coadd over %s", bbox)
967  tempExpName = self.getTempExpDatasetName(self.warpType)
968  coaddExposure.mask.addMaskPlane("REJECTED")
969  coaddExposure.mask.addMaskPlane("CLIPPED")
970  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
971  maskMap = self.setRejectedMaskMapping(statsCtrl)
972  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
973  maskedImageList = []
974  if nImage is not None:
975  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
976  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
977 
978  if isinstance(tempExpRef, DeferredDatasetHandle):
979  # Gen 3 API
980  exposure = tempExpRef.get(parameters={'bbox': bbox})
981  else:
982  # Gen 2 API. Delete this when Gen 2 retired
983  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
984 
985  maskedImage = exposure.getMaskedImage()
986  mask = maskedImage.getMask()
987  if altMask is not None:
988  self.applyAltMaskPlanes(mask, altMask)
989  imageScaler.scaleMaskedImage(maskedImage)
990 
991  # Add 1 for each pixel which is not excluded by the exclude mask.
992  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
993  if nImage is not None:
994  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
995  if self.config.removeMaskPlanes:
996  self.removeMaskPlanes(maskedImage)
997  maskedImageList.append(maskedImage)
998 
999  if self.config.doInputMap:
1000  visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1001  self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1002 
1003  with self.timer("stack"):
1004  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
1005  clipped, # also set output to CLIPPED if sigma-clipped
1006  maskMap)
1007  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
1008  if nImage is not None:
1009  nImage.assign(subNImage, bbox)
1010 
1011  def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList,
1012  altMaskList, statsCtrl, nImage=None):
1013  """Assemble the coadd using the "online" method.
1014 
1015  This method takes a running sum of images and weights to save memory.
1016  It only works for MEAN statistics.
1017 
1018  Parameters
1019  ----------
1020  coaddExposure : `lsst.afw.image.Exposure`
1021  The target exposure for the coadd.
1022  tempExpRefList : `list`
1023  List of data reference to tempExp.
1024  imageScalerList : `list`
1025  List of image scalers.
1026  weightList : `list`
1027  List of weights.
1028  altMaskList : `list`
1029  List of alternate masks to use rather than those stored with
1030  tempExp, or None. Each element is dict with keys = mask plane
1031  name to which to add the spans.
1032  statsCtrl : `lsst.afw.math.StatisticsControl`
1033  Statistics control object for coadd
1034  nImage : `lsst.afw.image.ImageU`, optional
1035  Keeps track of exposure count for each pixel.
1036  """
1037  self.log.debug("Computing online coadd.")
1038  tempExpName = self.getTempExpDatasetName(self.warpType)
1039  coaddExposure.mask.addMaskPlane("REJECTED")
1040  coaddExposure.mask.addMaskPlane("CLIPPED")
1041  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
1042  maskMap = self.setRejectedMaskMapping(statsCtrl)
1043  thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
1044 
1045  bbox = coaddExposure.maskedImage.getBBox()
1046 
1047  stacker = AccumulatorMeanStack(
1048  coaddExposure.image.array.shape,
1049  statsCtrl.getAndMask(),
1050  mask_threshold_dict=thresholdDict,
1051  mask_map=maskMap,
1052  no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
1053  calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
1054  compute_n_image=(nImage is not None)
1055  )
1056 
1057  for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList,
1058  imageScalerList,
1059  altMaskList,
1060  weightList):
1061  if isinstance(tempExpRef, DeferredDatasetHandle):
1062  # Gen 3 API
1063  exposure = tempExpRef.get()
1064  else:
1065  # Gen 2 API. Delete this when Gen 2 retired
1066  exposure = tempExpRef.get(tempExpName)
1067 
1068  maskedImage = exposure.getMaskedImage()
1069  mask = maskedImage.getMask()
1070  if altMask is not None:
1071  self.applyAltMaskPlanes(mask, altMask)
1072  imageScaler.scaleMaskedImage(maskedImage)
1073  if self.config.removeMaskPlanes:
1074  self.removeMaskPlanes(maskedImage)
1075 
1076  stacker.add_masked_image(maskedImage, weight=weight)
1077 
1078  if self.config.doInputMap:
1079  visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1080  self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1081 
1082  stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
1083 
1084  if nImage is not None:
1085  nImage.array[:, :] = stacker.n_image
1086 
1087  def removeMaskPlanes(self, maskedImage):
1088  """Unset the mask of an image for mask planes specified in the config.
1089 
1090  Parameters
1091  ----------
1092  maskedImage : `lsst.afw.image.MaskedImage`
1093  The masked image to be modified.
1094  """
1095  mask = maskedImage.getMask()
1096  for maskPlane in self.config.removeMaskPlanes:
1097  try:
1098  mask &= ~mask.getPlaneBitMask(maskPlane)
1099  except pexExceptions.InvalidParameterError:
1100  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
1101  maskPlane)
1102 
1103  @staticmethod
1104  def setRejectedMaskMapping(statsCtrl):
1105  """Map certain mask planes of the warps to new planes for the coadd.
1106 
1107  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1108  or CLIPPED, set it to REJECTED on the coadd.
1109  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1110  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1111 
1112  Parameters
1113  ----------
1114  statsCtrl : `lsst.afw.math.StatisticsControl`
1115  Statistics control object for coadd
1116 
1117  Returns
1118  -------
1119  maskMap : `list` of `tuple` of `int`
1120  A list of mappings of mask planes of the warped exposures to
1121  mask planes of the coadd.
1122  """
1123  edge = afwImage.Mask.getPlaneBitMask("EDGE")
1124  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1125  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1126  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1127  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1128  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1129  (clipped, clipped)]
1130  return maskMap
1131 
1132  def applyAltMaskPlanes(self, mask, altMaskSpans):
1133  """Apply in place alt mask formatted as SpanSets to a mask.
1134 
1135  Parameters
1136  ----------
1137  mask : `lsst.afw.image.Mask`
1138  Original mask.
1139  altMaskSpans : `dict`
1140  SpanSet lists to apply. Each element contains the new mask
1141  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1142  and list of SpanSets to apply to the mask.
1143 
1144  Returns
1145  -------
1146  mask : `lsst.afw.image.Mask`
1147  Updated mask.
1148  """
1149  if self.config.doUsePsfMatchedPolygons:
1150  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1151  # Clear away any other masks outside the validPolygons. These pixels are no longer
1152  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1153  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1154  # This mask-clearing step must only occur *before* applying the new masks below
1155  for spanSet in altMaskSpans['NO_DATA']:
1156  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1157 
1158  for plane, spanSetList in altMaskSpans.items():
1159  maskClipValue = mask.addMaskPlane(plane)
1160  for spanSet in spanSetList:
1161  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1162  return mask
1163 
1164  def shrinkValidPolygons(self, coaddInputs):
1165  """Shrink coaddInputs' ccds' ValidPolygons in place.
1166 
1167  Either modify each ccd's validPolygon in place, or if CoaddInputs
1168  does not have a validPolygon, create one from its bbox.
1169 
1170  Parameters
1171  ----------
1172  coaddInputs : `lsst.afw.image.coaddInputs`
1173  Original mask.
1174 
1175  """
1176  for ccd in coaddInputs.ccds:
1177  polyOrig = ccd.getValidPolygon()
1178  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1179  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1180  if polyOrig:
1181  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1182  else:
1183  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1184  ccd.setValidPolygon(validPolygon)
1185 
1186  def readBrightObjectMasks(self, dataRef):
1187  """Retrieve the bright object masks.
1188 
1189  Returns None on failure.
1190 
1191  Parameters
1192  ----------
1193  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1194  A Butler dataRef.
1195 
1196  Returns
1197  -------
1198  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1199  Bright object mask from the Butler object, or None if it cannot
1200  be retrieved.
1201  """
1202  try:
1203  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1204  except Exception as e:
1205  self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1206  return None
1207 
1208  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1209  """Set the bright object masks.
1210 
1211  Parameters
1212  ----------
1213  exposure : `lsst.afw.image.Exposure`
1214  Exposure under consideration.
1215  dataId : `lsst.daf.persistence.dataId`
1216  Data identifier dict for patch.
1217  brightObjectMasks : `lsst.afw.table`
1218  Table of bright objects to mask.
1219  """
1220 
1221  if brightObjectMasks is None:
1222  self.log.warning("Unable to apply bright object mask: none supplied")
1223  return
1224  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1225  mask = exposure.getMaskedImage().getMask()
1226  wcs = exposure.getWcs()
1227  plateScale = wcs.getPixelScale().asArcseconds()
1228 
1229  for rec in brightObjectMasks:
1230  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1231  if rec["type"] == "box":
1232  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1233  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1234  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1235 
1236  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1237  bbox = geom.Box2I(center - halfSize, center + halfSize)
1238 
1239  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1240  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1241  spans = afwGeom.SpanSet(bbox)
1242  elif rec["type"] == "circle":
1243  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1244  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1245  else:
1246  self.log.warning("Unexpected region type %s at %s", rec["type"], center)
1247  continue
1248  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1249 
1250  def setInexactPsf(self, mask):
1251  """Set INEXACT_PSF mask plane.
1252 
1253  If any of the input images isn't represented in the coadd (due to
1254  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1255  these pixels.
1256 
1257  Parameters
1258  ----------
1259  mask : `lsst.afw.image.Mask`
1260  Coadded exposure's mask, modified in-place.
1261  """
1262  mask.addMaskPlane("INEXACT_PSF")
1263  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1264  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1265  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1266  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1267  array = mask.getArray()
1268  selected = array & (sensorEdge | clipped | rejected) > 0
1269  array[selected] |= inexactPsf
1270 
1271  @classmethod
1272  def _makeArgumentParser(cls):
1273  """Create an argument parser.
1274  """
1275  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1276  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1277  + cls.ConfigClass().warpType + "Warp",
1278  help="data ID, e.g. --id tract=12345 patch=1,2",
1279  ContainerClass=AssembleCoaddDataIdContainer)
1280  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1281  ContainerClass=SelectDataIdContainer)
1282  return parser
1283 
1284  @staticmethod
1285  def _subBBoxIter(bbox, subregionSize):
1286  """Iterate over subregions of a bbox.
1287 
1288  Parameters
1289  ----------
1290  bbox : `lsst.geom.Box2I`
1291  Bounding box over which to iterate.
1292  subregionSize: `lsst.geom.Extent2I`
1293  Size of sub-bboxes.
1294 
1295  Yields
1296  ------
1297  subBBox : `lsst.geom.Box2I`
1298  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1299  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1300  the edges of ``bbox``, but it will never be empty.
1301  """
1302  if bbox.isEmpty():
1303  raise RuntimeError("bbox %s is empty" % (bbox,))
1304  if subregionSize[0] < 1 or subregionSize[1] < 1:
1305  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1306 
1307  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1308  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1309  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1310  subBBox.clip(bbox)
1311  if subBBox.isEmpty():
1312  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1313  "colShift=%s, rowShift=%s" %
1314  (bbox, subregionSize, colShift, rowShift))
1315  yield subBBox
1316 
1317  def filterWarps(self, inputs, goodVisits):
1318  """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1319 
1320  Parameters
1321  ----------
1322  inputs : list
1323  List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1324  goodVisit : `dict`
1325  Dictionary with good visitIds as the keys. Value ignored.
1326 
1327  Returns:
1328  --------
1329  filteredInputs : `list`
1330  Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1331  """
1332  inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1333  filteredInputs = []
1334  for visit in goodVisits.keys():
1335  filteredInputs.append(inputWarpDict[visit])
1336  return filteredInputs
1337 
1338 
1339 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1340  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1341  """
1342 
1343  def makeDataRefList(self, namespace):
1344  """Make self.refList from self.idList.
1345 
1346  Parameters
1347  ----------
1348  namespace
1349  Results of parsing command-line (with ``butler`` and ``log`` elements).
1350  """
1351  datasetType = namespace.config.coaddName + "Coadd"
1352  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1353 
1354  for dataId in self.idList:
1355  # tract and patch are required
1356  for key in keysCoadd:
1357  if key not in dataId:
1358  raise RuntimeError("--id must include " + key)
1359 
1360  dataRef = namespace.butler.dataRef(
1361  datasetType=datasetType,
1362  dataId=dataId,
1363  )
1364  self.refList.append(dataRef)
1365 
1366 
1367 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1368  """Function to count the number of pixels with a specific mask in a
1369  footprint.
1370 
1371  Find the intersection of mask & footprint. Count all pixels in the mask
1372  that are in the intersection that have bitmask set but do not have
1373  ignoreMask set. Return the count.
1374 
1375  Parameters
1376  ----------
1377  mask : `lsst.afw.image.Mask`
1378  Mask to define intersection region by.
1379  footprint : `lsst.afw.detection.Footprint`
1380  Footprint to define the intersection region by.
1381  bitmask
1382  Specific mask that we wish to count the number of occurances of.
1383  ignoreMask
1384  Pixels to not consider.
1385 
1386  Returns
1387  -------
1388  result : `int`
1389  Count of number of pixels in footprint with specified mask.
1390  """
1391  bbox = footprint.getBBox()
1392  bbox.clip(mask.getBBox(afwImage.PARENT))
1393  fp = afwImage.Mask(bbox)
1394  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1395  footprint.spans.setMask(fp, bitmask)
1396  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1397  (subMask.getArray() & ignoreMask) == 0).sum()
1398 
1399 
1400 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1401  """Configuration parameters for the SafeClipAssembleCoaddTask.
1402  """
1403  clipDetection = pexConfig.ConfigurableField(
1404  target=SourceDetectionTask,
1405  doc="Detect sources on difference between unclipped and clipped coadd")
1406  minClipFootOverlap = pexConfig.Field(
1407  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1408  dtype=float,
1409  default=0.6
1410  )
1411  minClipFootOverlapSingle = pexConfig.Field(
1412  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1413  "clipped when only one visit overlaps",
1414  dtype=float,
1415  default=0.5
1416  )
1417  minClipFootOverlapDouble = pexConfig.Field(
1418  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1419  "clipped when two visits overlap",
1420  dtype=float,
1421  default=0.45
1422  )
1423  maxClipFootOverlapDouble = pexConfig.Field(
1424  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1425  "considering two visits",
1426  dtype=float,
1427  default=0.15
1428  )
1429  minBigOverlap = pexConfig.Field(
1430  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1431  "when labeling clipped footprints",
1432  dtype=int,
1433  default=100
1434  )
1435 
1436  def setDefaults(self):
1437  """Set default values for clipDetection.
1438 
1439  Notes
1440  -----
1441  The numeric values for these configuration parameters were
1442  empirically determined, future work may further refine them.
1443  """
1444  AssembleCoaddConfig.setDefaults(self)
1445  self.clipDetectionclipDetection.doTempLocalBackground = False
1446  self.clipDetectionclipDetection.reEstimateBackground = False
1447  self.clipDetectionclipDetection.returnOriginalFootprints = False
1448  self.clipDetectionclipDetection.thresholdPolarity = "both"
1449  self.clipDetectionclipDetection.thresholdValue = 2
1450  self.clipDetectionclipDetection.nSigmaToGrow = 2
1451  self.clipDetectionclipDetection.minPixels = 4
1452  self.clipDetectionclipDetection.isotropicGrow = True
1453  self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1454  self.sigmaClipsigmaClip = 1.5
1455  self.clipIterclipIter = 3
1456  self.statisticstatistic = "MEAN"
1457 
1458  def validate(self):
1459  if self.doSigmaClipdoSigmaClip:
1460  log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1461  "Ignoring doSigmaClip.")
1462  self.doSigmaClipdoSigmaClip = False
1463  if self.statisticstatistic != "MEAN":
1464  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1465  "(%s chosen). Please set statistic to MEAN."
1466  % (self.statisticstatistic))
1467  AssembleCoaddTask.ConfigClass.validate(self)
1468 
1469 
1470 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1471  """Assemble a coadded image from a set of coadded temporary exposures,
1472  being careful to clip & flag areas with potential artifacts.
1473 
1474  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1475  we clip outliers). The problem with doing this is that when computing the
1476  coadd PSF at a given location, individual visit PSFs from visits with
1477  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1478  In this task, we correct for this behavior by creating a new
1479  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1480  coaddTempExps and the final coadd where
1481 
1482  i. difference imaging suggests that there is an outlier and
1483  ii. this outlier appears on only one or two images.
1484 
1485  Such regions will not contribute to the final coadd. Furthermore, any
1486  routine to determine the coadd PSF can now be cognizant of clipped regions.
1487  Note that the algorithm implemented by this task is preliminary and works
1488  correctly for HSC data. Parameter modifications and or considerable
1489  redesigning of the algorithm is likley required for other surveys.
1490 
1491  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1492  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1493  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1494  if you wish.
1495 
1496  Notes
1497  -----
1498  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1499  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1500  see `baseDebug` for more about ``debug.py`` files.
1501  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1502  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1503  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1504  for further information.
1505 
1506  Examples
1507  --------
1508  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1509  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1510  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1511 
1512  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1513  and filter to be coadded (specified using
1514  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1515  along with a list of coaddTempExps to attempt to coadd (specified using
1516  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1517  Only the coaddTempExps that cover the specified tract and patch will be
1518  coadded. A list of the available optional arguments can be obtained by
1519  calling assembleCoadd.py with the --help command line argument:
1520 
1521  .. code-block:: none
1522 
1523  assembleCoadd.py --help
1524 
1525  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1526  context of multi-band processing, we will generate the HSC-I & -R band
1527  coadds from HSC engineering test data provided in the ci_hsc package.
1528  To begin, assuming that the lsst stack has been already set up, we must
1529  set up the obs_subaru and ci_hsc packages. This defines the environment
1530  variable $CI_HSC_DIR and points at the location of the package. The raw
1531  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1532  the coadds, we must first
1533 
1534  - ``processCcd``
1535  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1536  - ``makeSkyMap``
1537  create a skymap that covers the area of the sky present in the raw exposures
1538  - ``makeCoaddTempExp``
1539  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1540 
1541  We can perform all of these steps by running
1542 
1543  .. code-block:: none
1544 
1545  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1546 
1547  This will produce warped coaddTempExps for each visit. To coadd the
1548  warped data, we call ``assembleCoadd.py`` as follows:
1549 
1550  .. code-block:: none
1551 
1552  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1553  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1554  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1555  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1556  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1557  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1558  --selectId visit=903988 ccd=24
1559 
1560  This will process the HSC-I band data. The results are written in
1561  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1562 
1563  You may also choose to run:
1564 
1565  .. code-block:: none
1566 
1567  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1568  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1569  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1570  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1571  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1572  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1573  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1574  --selectId visit=903346 ccd=12
1575 
1576  to generate the coadd for the HSC-R band if you are interested in following
1577  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1578  """
1579  ConfigClass = SafeClipAssembleCoaddConfig
1580  _DefaultName = "safeClipAssembleCoadd"
1581 
1582  def __init__(self, *args, **kwargs):
1583  AssembleCoaddTask.__init__(self, *args, **kwargs)
1584  schema = afwTable.SourceTable.makeMinimalSchema()
1585  self.makeSubtask("clipDetection", schema=schema)
1586 
1587  @utils.inheritDoc(AssembleCoaddTask)
1588  @pipeBase.timeMethod
1589  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1590  """Assemble the coadd for a region.
1591 
1592  Compute the difference of coadds created with and without outlier
1593  rejection to identify coadd pixels that have outlier values in some
1594  individual visits.
1595  Detect clipped regions on the difference image and mark these regions
1596  on the one or two individual coaddTempExps where they occur if there
1597  is significant overlap between the clipped region and a source. This
1598  leaves us with a set of footprints from the difference image that have
1599  been identified as having occured on just one or two individual visits.
1600  However, these footprints were generated from a difference image. It
1601  is conceivable for a large diffuse source to have become broken up
1602  into multiple footprints acrosss the coadd difference in this process.
1603  Determine the clipped region from all overlapping footprints from the
1604  detected sources in each visit - these are big footprints.
1605  Combine the small and big clipped footprints and mark them on a new
1606  bad mask plane.
1607  Generate the coadd using `AssembleCoaddTask.run` without outlier
1608  removal. Clipped footprints will no longer make it into the coadd
1609  because they are marked in the new bad mask plane.
1610 
1611  Notes
1612  -----
1613  args and kwargs are passed but ignored in order to match the call
1614  signature expected by the parent task.
1615  """
1616  exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1617  mask = exp.getMaskedImage().getMask()
1618  mask.addMaskPlane("CLIPPED")
1619 
1620  result = self.detectClipdetectClip(exp, tempExpRefList)
1621 
1622  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1623 
1624  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1625  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1626  # Append big footprints from individual Warps to result.clipSpans
1627  bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1628  result.detectionFootprints, maskClipValue, maskDetValue,
1629  exp.getBBox())
1630  # Create mask of the current clipped footprints
1631  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1632  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1633 
1634  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1635  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1636  maskClip |= maskClipBig
1637 
1638  # Assemble coadd from base class, but ignoring CLIPPED pixels
1639  badMaskPlanes = self.config.badMaskPlanes[:]
1640  badMaskPlanes.append("CLIPPED")
1641  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1642  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1643  result.clipSpans, mask=badPixelMask)
1644 
1645  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1646  """Return an exposure that contains the difference between unclipped
1647  and clipped coadds.
1648 
1649  Generate a difference image between clipped and unclipped coadds.
1650  Compute the difference image by subtracting an outlier-clipped coadd
1651  from an outlier-unclipped coadd. Return the difference image.
1652 
1653  Parameters
1654  ----------
1655  skyInfo : `lsst.pipe.base.Struct`
1656  Patch geometry information, from getSkyInfo
1657  tempExpRefList : `list`
1658  List of data reference to tempExp
1659  imageScalerList : `list`
1660  List of image scalers
1661  weightList : `list`
1662  List of weights
1663 
1664  Returns
1665  -------
1666  exp : `lsst.afw.image.Exposure`
1667  Difference image of unclipped and clipped coadd wrapped in an Exposure
1668  """
1669  config = AssembleCoaddConfig()
1670  # getattr necessary because subtasks do not survive Config.toDict()
1671  # exclude connections because the class of self.config.connections is not
1672  # the same as AssembleCoaddConfig.connections, and the connections are not
1673  # needed to run this task anyway.
1674  configIntersection = {k: getattr(self.config, k)
1675  for k, v in self.config.toDict().items()
1676  if (k in config.keys() and k != "connections")}
1677  configIntersection['doInputMap'] = False
1678  configIntersection['doNImage'] = False
1679  config.update(**configIntersection)
1680 
1681  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1682  config.statistic = 'MEAN'
1683  task = AssembleCoaddTask(config=config)
1684  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1685 
1686  config.statistic = 'MEANCLIP'
1687  task = AssembleCoaddTask(config=config)
1688  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1689 
1690  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1691  coaddDiff -= coaddClip.getMaskedImage()
1692  exp = afwImage.ExposureF(coaddDiff)
1693  exp.setPsf(coaddMean.getPsf())
1694  return exp
1695 
1696  def detectClip(self, exp, tempExpRefList):
1697  """Detect clipped regions on an exposure and set the mask on the
1698  individual tempExp masks.
1699 
1700  Detect footprints in the difference image after smoothing the
1701  difference image with a Gaussian kernal. Identify footprints that
1702  overlap with one or two input ``coaddTempExps`` by comparing the
1703  computed overlap fraction to thresholds set in the config. A different
1704  threshold is applied depending on the number of overlapping visits
1705  (restricted to one or two). If the overlap exceeds the thresholds,
1706  the footprint is considered "CLIPPED" and is marked as such on the
1707  coaddTempExp. Return a struct with the clipped footprints, the indices
1708  of the ``coaddTempExps`` that end up overlapping with the clipped
1709  footprints, and a list of new masks for the ``coaddTempExps``.
1710 
1711  Parameters
1712  ----------
1713  exp : `lsst.afw.image.Exposure`
1714  Exposure to run detection on.
1715  tempExpRefList : `list`
1716  List of data reference to tempExp.
1717 
1718  Returns
1719  -------
1720  result : `lsst.pipe.base.Struct`
1721  Result struct with components:
1722 
1723  - ``clipFootprints``: list of clipped footprints.
1724  - ``clipIndices``: indices for each ``clippedFootprint`` in
1725  ``tempExpRefList``.
1726  - ``clipSpans``: List of dictionaries containing spanSet lists
1727  to clip. Each element contains the new maskplane name
1728  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1729  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1730  compressed into footprints.
1731  """
1732  mask = exp.getMaskedImage().getMask()
1733  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1734  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1735  # Merge positive and negative together footprints together
1736  fpSet.positive.merge(fpSet.negative)
1737  footprints = fpSet.positive
1738  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1739  ignoreMask = self.getBadPixelMask()
1740 
1741  clipFootprints = []
1742  clipIndices = []
1743  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1744 
1745  # for use by detectClipBig
1746  visitDetectionFootprints = []
1747 
1748  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1749  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1750  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1751 
1752  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1753  for i, warpRef in enumerate(tempExpRefList):
1754  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1755  immediate=True).getMaskedImage().getMask()
1756  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1757  afwImage.PARENT, True)
1758  maskVisitDet &= maskDetValue
1759  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1760  visitDetectionFootprints.append(visitFootprints)
1761 
1762  for j, footprint in enumerate(footprints.getFootprints()):
1763  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1764  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1765 
1766  # build a list of clipped spans for each visit
1767  for j, footprint in enumerate(footprints.getFootprints()):
1768  nPixel = footprint.getArea()
1769  overlap = [] # hold the overlap with each visit
1770  indexList = [] # index of visit in global list
1771  for i in range(len(tempExpRefList)):
1772  ignore = ignoreArr[i, j]
1773  overlapDet = overlapDetArr[i, j]
1774  totPixel = nPixel - ignore
1775 
1776  # If we have more bad pixels than detection skip
1777  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1778  continue
1779  overlap.append(overlapDet/float(totPixel))
1780  indexList.append(i)
1781 
1782  overlap = numpy.array(overlap)
1783  if not len(overlap):
1784  continue
1785 
1786  keep = False # Should this footprint be marked as clipped?
1787  keepIndex = [] # Which tempExps does the clipped footprint belong to
1788 
1789  # If footprint only has one overlap use a lower threshold
1790  if len(overlap) == 1:
1791  if overlap[0] > self.config.minClipFootOverlapSingle:
1792  keep = True
1793  keepIndex = [0]
1794  else:
1795  # This is the general case where only visit should be clipped
1796  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1797  if len(clipIndex) == 1:
1798  keep = True
1799  keepIndex = [clipIndex[0]]
1800 
1801  # Test if there are clipped objects that overlap two different visits
1802  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1803  if len(clipIndex) == 2 and len(overlap) > 3:
1804  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1805  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1806  keep = True
1807  keepIndex = clipIndex
1808 
1809  if not keep:
1810  continue
1811 
1812  for index in keepIndex:
1813  globalIndex = indexList[index]
1814  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1815 
1816  clipIndices.append(numpy.array(indexList)[keepIndex])
1817  clipFootprints.append(footprint)
1818 
1819  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1820  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1821 
1822  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1823  maskClipValue, maskDetValue, coaddBBox):
1824  """Return individual warp footprints for large artifacts and append
1825  them to ``clipList`` in place.
1826 
1827  Identify big footprints composed of many sources in the coadd
1828  difference that may have originated in a large diffuse source in the
1829  coadd. We do this by indentifying all clipped footprints that overlap
1830  significantly with each source in all the coaddTempExps.
1831 
1832  Parameters
1833  ----------
1834  clipList : `list`
1835  List of alt mask SpanSets with clipping information. Modified.
1836  clipFootprints : `list`
1837  List of clipped footprints.
1838  clipIndices : `list`
1839  List of which entries in tempExpClipList each footprint belongs to.
1840  maskClipValue
1841  Mask value of clipped pixels.
1842  maskDetValue
1843  Mask value of detected pixels.
1844  coaddBBox : `lsst.geom.Box`
1845  BBox of the coadd and warps.
1846 
1847  Returns
1848  -------
1849  bigFootprintsCoadd : `list`
1850  List of big footprints
1851  """
1852  bigFootprintsCoadd = []
1853  ignoreMask = self.getBadPixelMask()
1854  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1855  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1856  for footprint in visitFootprints.getFootprints():
1857  footprint.spans.setMask(maskVisitDet, maskDetValue)
1858 
1859  # build a mask of clipped footprints that are in this visit
1860  clippedFootprintsVisit = []
1861  for foot, clipIndex in zip(clipFootprints, clipIndices):
1862  if index not in clipIndex:
1863  continue
1864  clippedFootprintsVisit.append(foot)
1865  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1866  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1867 
1868  bigFootprintsVisit = []
1869  for foot in visitFootprints.getFootprints():
1870  if foot.getArea() < self.config.minBigOverlap:
1871  continue
1872  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1873  if nCount > self.config.minBigOverlap:
1874  bigFootprintsVisit.append(foot)
1875  bigFootprintsCoadd.append(foot)
1876 
1877  for footprint in bigFootprintsVisit:
1878  clippedSpans["CLIPPED"].append(footprint.spans)
1879 
1880  return bigFootprintsCoadd
1881 
1882 
1884  psfMatchedWarps = pipeBase.connectionTypes.Input(
1885  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1886  "Only PSF-Matched Warps make sense for image subtraction. "
1887  "Therefore, they must be an additional declared input."),
1888  name="{inputCoaddName}Coadd_psfMatchedWarp",
1889  storageClass="ExposureF",
1890  dimensions=("tract", "patch", "skymap", "visit"),
1891  deferLoad=True,
1892  multiple=True
1893  )
1894  templateCoadd = pipeBase.connectionTypes.Output(
1895  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1896  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1897  name="{outputCoaddName}CoaddPsfMatched",
1898  storageClass="ExposureF",
1899  dimensions=("tract", "patch", "skymap", "band"),
1900  )
1901 
1902  def __init__(self, *, config=None):
1903  super().__init__(config=config)
1904  if not config.assembleStaticSkyModel.doWrite:
1905  self.outputs.remove("templateCoadd")
1906  config.validate()
1907 
1908 
1909 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1910  pipelineConnections=CompareWarpAssembleCoaddConnections):
1911  assembleStaticSkyModel = pexConfig.ConfigurableField(
1912  target=AssembleCoaddTask,
1913  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1914  " naive/first-iteration model of the static sky.",
1915  )
1916  detect = pexConfig.ConfigurableField(
1917  target=SourceDetectionTask,
1918  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1919  )
1920  detectTemplate = pexConfig.ConfigurableField(
1921  target=SourceDetectionTask,
1922  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1923  )
1924  maskStreaks = pexConfig.ConfigurableField(
1925  target=MaskStreaksTask,
1926  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1927  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1928  "streakMaskName"
1929  )
1930  streakMaskName = pexConfig.Field(
1931  dtype=str,
1932  default="STREAK",
1933  doc="Name of mask bit used for streaks"
1934  )
1935  maxNumEpochs = pexConfig.Field(
1936  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1937  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1938  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1939  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1940  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1941  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1942  "than transient and not masked.",
1943  dtype=int,
1944  default=2
1945  )
1946  maxFractionEpochsLow = pexConfig.RangeField(
1947  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1948  "Effective maxNumEpochs = "
1949  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1950  dtype=float,
1951  default=0.4,
1952  min=0., max=1.,
1953  )
1954  maxFractionEpochsHigh = pexConfig.RangeField(
1955  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1956  "Effective maxNumEpochs = "
1957  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1958  dtype=float,
1959  default=0.03,
1960  min=0., max=1.,
1961  )
1962  spatialThreshold = pexConfig.RangeField(
1963  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1964  "temporal criteria. If 0, clip all. If 1, clip none.",
1965  dtype=float,
1966  default=0.5,
1967  min=0., max=1.,
1968  inclusiveMin=True, inclusiveMax=True
1969  )
1970  doScaleWarpVariance = pexConfig.Field(
1971  doc="Rescale Warp variance plane using empirical noise?",
1972  dtype=bool,
1973  default=True,
1974  )
1975  scaleWarpVariance = pexConfig.ConfigurableField(
1976  target=ScaleVarianceTask,
1977  doc="Rescale variance on warps",
1978  )
1979  doPreserveContainedBySource = pexConfig.Field(
1980  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1981  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1982  dtype=bool,
1983  default=True,
1984  )
1985  doPrefilterArtifacts = pexConfig.Field(
1986  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1987  "because they will be excluded anyway. This prevents them from contributing "
1988  "to the outlier epoch count image and potentially being labeled as persistant."
1989  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1990  dtype=bool,
1991  default=True
1992  )
1993  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1994  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1995  dtype=str,
1996  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1997  )
1998  prefilterArtifactsRatio = pexConfig.Field(
1999  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
2000  dtype=float,
2001  default=0.05
2002  )
2003  doFilterMorphological = pexConfig.Field(
2004  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
2005  "be streaks.",
2006  dtype=bool,
2007  default=False
2008  )
2009 
2010  def setDefaults(self):
2011  AssembleCoaddConfig.setDefaults(self)
2012  self.statisticstatistic = 'MEAN'
2013  self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
2014 
2015  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
2016  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
2017  if "EDGE" in self.badMaskPlanes:
2018  self.badMaskPlanes.remove('EDGE')
2019  self.removeMaskPlanes.append('EDGE')
2020  self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
2021  self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
2022  self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
2023  self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
2024  self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
2025  self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
2026  self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
2027  self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
2028  self.detectdetect.doTempLocalBackground = False
2029  self.detectdetect.reEstimateBackground = False
2030  self.detectdetect.returnOriginalFootprints = False
2031  self.detectdetect.thresholdPolarity = "both"
2032  self.detectdetect.thresholdValue = 5
2033  self.detectdetect.minPixels = 4
2034  self.detectdetect.isotropicGrow = True
2035  self.detectdetect.thresholdType = "pixel_stdev"
2036  self.detectdetect.nSigmaToGrow = 0.4
2037  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
2038  # Explicitly restating because ratio with detect.nSigmaToGrow matters
2039  self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
2040  self.detectTemplatedetectTemplate.doTempLocalBackground = False
2041  self.detectTemplatedetectTemplate.reEstimateBackground = False
2042  self.detectTemplatedetectTemplate.returnOriginalFootprints = False
2043 
2044  def validate(self):
2045  super().validate()
2046  if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
2047  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
2048  "Please set assembleStaticSkyModel.doNImage=False")
2049 
2050  if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
2051  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
2052  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
2053  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
2054  "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
2055 
2056 
2057 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
2058  """Assemble a compareWarp coadded image from a set of warps
2059  by masking artifacts detected by comparing PSF-matched warps.
2060 
2061  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
2062  we clip outliers). The problem with doing this is that when computing the
2063  coadd PSF at a given location, individual visit PSFs from visits with
2064  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
2065  In this task, we correct for this behavior by creating a new badMaskPlane
2066  'CLIPPED' which marks pixels in the individual warps suspected to contain
2067  an artifact. We populate this plane on the input warps by comparing
2068  PSF-matched warps with a PSF-matched median coadd which serves as a
2069  model of the static sky. Any group of pixels that deviates from the
2070  PSF-matched template coadd by more than config.detect.threshold sigma,
2071  is an artifact candidate. The candidates are then filtered to remove
2072  variable sources and sources that are difficult to subtract such as
2073  bright stars. This filter is configured using the config parameters
2074  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
2075  the maximum fraction of epochs that the deviation can appear in and still
2076  be considered an artifact. The spatialThreshold is the maximum fraction of
2077  pixels in the footprint of the deviation that appear in other epochs
2078  (where other epochs is defined by the temporalThreshold). If the deviant
2079  region meets this criteria of having a significant percentage of pixels
2080  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
2081  set in the mask. These regions will not contribute to the final coadd.
2082  Furthermore, any routine to determine the coadd PSF can now be cognizant
2083  of clipped regions. Note that the algorithm implemented by this task is
2084  preliminary and works correctly for HSC data. Parameter modifications and
2085  or considerable redesigning of the algorithm is likley required for other
2086  surveys.
2087 
2088  ``CompareWarpAssembleCoaddTask`` sub-classes
2089  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
2090  as a subtask to generate the TemplateCoadd (the model of the static sky).
2091 
2092  Notes
2093  -----
2094  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2095  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2096  ``baseDebug`` for more about ``debug.py`` files.
2097 
2098  This task supports the following debug variables:
2099 
2100  - ``saveCountIm``
2101  If True then save the Epoch Count Image as a fits file in the `figPath`
2102  - ``figPath``
2103  Path to save the debug fits images and figures
2104 
2105  For example, put something like:
2106 
2107  .. code-block:: python
2108 
2109  import lsstDebug
2110  def DebugInfo(name):
2111  di = lsstDebug.getInfo(name)
2112  if name == "lsst.pipe.tasks.assembleCoadd":
2113  di.saveCountIm = True
2114  di.figPath = "/desired/path/to/debugging/output/images"
2115  return di
2116  lsstDebug.Info = DebugInfo
2117 
2118  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2119  ``--debug`` flag. Some subtasks may have their own debug variables;
2120  see individual Task documentation.
2121 
2122  Examples
2123  --------
2124  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2125  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2126  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2127  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2128  and filter to be coadded (specified using
2129  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2130  along with a list of coaddTempExps to attempt to coadd (specified using
2131  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2132  Only the warps that cover the specified tract and patch will be coadded.
2133  A list of the available optional arguments can be obtained by calling
2134  ``assembleCoadd.py`` with the ``--help`` command line argument:
2135 
2136  .. code-block:: none
2137 
2138  assembleCoadd.py --help
2139 
2140  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2141  context of multi-band processing, we will generate the HSC-I & -R band
2142  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2143  To begin, assuming that the lsst stack has been already set up, we must
2144  set up the ``obs_subaru`` and ``ci_hsc`` packages.
2145  This defines the environment variable ``$CI_HSC_DIR`` and points at the
2146  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2147  directory. To begin assembling the coadds, we must first
2148 
2149  - processCcd
2150  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2151  - makeSkyMap
2152  create a skymap that covers the area of the sky present in the raw exposures
2153  - makeCoaddTempExp
2154  warp the individual calibrated exposures to the tangent plane of the coadd
2155 
2156  We can perform all of these steps by running
2157 
2158  .. code-block:: none
2159 
2160  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2161 
2162  This will produce warped ``coaddTempExps`` for each visit. To coadd the
2163  warped data, we call ``assembleCoadd.py`` as follows:
2164 
2165  .. code-block:: none
2166 
2167  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2168  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2169  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2170  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2171  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2172  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2173  --selectId visit=903988 ccd=24
2174 
2175  This will process the HSC-I band data. The results are written in
2176  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2177  """
2178  ConfigClass = CompareWarpAssembleCoaddConfig
2179  _DefaultName = "compareWarpAssembleCoadd"
2180 
2181  def __init__(self, *args, **kwargs):
2182  AssembleCoaddTask.__init__(self, *args, **kwargs)
2183  self.makeSubtask("assembleStaticSkyModel")
2184  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2185  self.makeSubtask("detect", schema=detectionSchema)
2186  if self.config.doPreserveContainedBySource:
2187  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2188  if self.config.doScaleWarpVariance:
2189  self.makeSubtask("scaleWarpVariance")
2190  if self.config.doFilterMorphological:
2191  self.makeSubtask("maskStreaks")
2192 
2193  @utils.inheritDoc(AssembleCoaddTask)
2194  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2195  """
2196  Generate a templateCoadd to use as a naive model of static sky to
2197  subtract from PSF-Matched warps.
2198 
2199  Returns
2200  -------
2201  result : `lsst.pipe.base.Struct`
2202  Result struct with components:
2203 
2204  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2205  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2206  """
2207  # Ensure that psfMatchedWarps are used as input warps for template generation
2208  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2209  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2210 
2211  # Because subtasks don't have connections we have to make one.
2212  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2213  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2214  if self.config.assembleStaticSkyModel.doWrite:
2215  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2216  # Remove template coadd from both subtask's and main tasks outputs,
2217  # because it is handled by the subtask as `coaddExposure`
2218  del outputRefs.templateCoadd
2219  del staticSkyModelOutputRefs.templateCoadd
2220 
2221  # A PSF-Matched nImage does not exist as a dataset type
2222  if 'nImage' in staticSkyModelOutputRefs.keys():
2223  del staticSkyModelOutputRefs.nImage
2224 
2225  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2226  staticSkyModelOutputRefs)
2227  if templateCoadd is None:
2228  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2229 
2230  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2231  nImage=templateCoadd.nImage,
2232  warpRefList=templateCoadd.warpRefList,
2233  imageScalerList=templateCoadd.imageScalerList,
2234  weightList=templateCoadd.weightList)
2235 
2236  @utils.inheritDoc(AssembleCoaddTask)
2237  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2238  """
2239  Generate a templateCoadd to use as a naive model of static sky to
2240  subtract from PSF-Matched warps.
2241 
2242  Returns
2243  -------
2244  result : `lsst.pipe.base.Struct`
2245  Result struct with components:
2246 
2247  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2248  - ``nImage``: N Image (``lsst.afw.image.Image``)
2249  """
2250  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2251  if templateCoadd is None:
2252  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2253 
2254  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2255  nImage=templateCoadd.nImage,
2256  warpRefList=templateCoadd.warpRefList,
2257  imageScalerList=templateCoadd.imageScalerList,
2258  weightList=templateCoadd.weightList)
2259 
2260  def _noTemplateMessage(self, warpType):
2261  warpName = (warpType[0].upper() + warpType[1:])
2262  message = """No %(warpName)s warps were found to build the template coadd which is
2263  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2264  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2265  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2266 
2267  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2268  another algorithm like:
2269 
2270  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2271  config.assemble.retarget(SafeClipAssembleCoaddTask)
2272  """ % {"warpName": warpName}
2273  return message
2274 
2275  @utils.inheritDoc(AssembleCoaddTask)
2276  @pipeBase.timeMethod
2277  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2278  supplementaryData, *args, **kwargs):
2279  """Assemble the coadd.
2280 
2281  Find artifacts and apply them to the warps' masks creating a list of
2282  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2283  plane. Then pass these alternative masks to the base class's `run`
2284  method.
2285 
2286  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2287  that must contain a ``templateCoadd`` that serves as the
2288  model of the static sky.
2289  """
2290 
2291  # Check and match the order of the supplementaryData
2292  # (PSF-matched) inputs to the order of the direct inputs,
2293  # so that the artifact mask is applied to the right warp
2294  dataIds = [ref.dataId for ref in tempExpRefList]
2295  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2296 
2297  if dataIds != psfMatchedDataIds:
2298  self.log.info("Reordering and or/padding PSF-matched visit input list")
2299  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2300  psfMatchedDataIds, dataIds)
2301  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2302  psfMatchedDataIds, dataIds)
2303 
2304  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2305  spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2306  supplementaryData.warpRefList,
2307  supplementaryData.imageScalerList)
2308 
2309  badMaskPlanes = self.config.badMaskPlanes[:]
2310  badMaskPlanes.append("CLIPPED")
2311  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2312 
2313  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2314  spanSetMaskList, mask=badPixelMask)
2315 
2316  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2317  # Psf-Matching moves the real edge inwards
2318  self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2319  return result
2320 
2321  def applyAltEdgeMask(self, mask, altMaskList):
2322  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2323 
2324  Parameters
2325  ----------
2326  mask : `lsst.afw.image.Mask`
2327  Original mask.
2328  altMaskList : `list`
2329  List of Dicts containing ``spanSet`` lists.
2330  Each element contains the new mask plane name (e.g. "CLIPPED
2331  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2332  the mask.
2333  """
2334  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2335  for visitMask in altMaskList:
2336  if "EDGE" in visitMask:
2337  for spanSet in visitMask['EDGE']:
2338  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2339 
2340  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2341  """Find artifacts.
2342 
2343  Loop through warps twice. The first loop builds a map with the count
2344  of how many epochs each pixel deviates from the templateCoadd by more
2345  than ``config.chiThreshold`` sigma. The second loop takes each
2346  difference image and filters the artifacts detected in each using
2347  count map to filter out variable sources and sources that are
2348  difficult to subtract cleanly.
2349 
2350  Parameters
2351  ----------
2352  templateCoadd : `lsst.afw.image.Exposure`
2353  Exposure to serve as model of static sky.
2354  tempExpRefList : `list`
2355  List of data references to warps.
2356  imageScalerList : `list`
2357  List of image scalers.
2358 
2359  Returns
2360  -------
2361  altMasks : `list`
2362  List of dicts containing information about CLIPPED
2363  (i.e., artifacts), NO_DATA, and EDGE pixels.
2364  """
2365 
2366  self.log.debug("Generating Count Image, and mask lists.")
2367  coaddBBox = templateCoadd.getBBox()
2368  slateIm = afwImage.ImageU(coaddBBox)
2369  epochCountImage = afwImage.ImageU(coaddBBox)
2370  nImage = afwImage.ImageU(coaddBBox)
2371  spanSetArtifactList = []
2372  spanSetNoDataMaskList = []
2373  spanSetEdgeList = []
2374  spanSetBadMorphoList = []
2375  badPixelMask = self.getBadPixelMask()
2376 
2377  # mask of the warp diffs should = that of only the warp
2378  templateCoadd.mask.clearAllMaskPlanes()
2379 
2380  if self.config.doPreserveContainedBySource:
2381  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2382  else:
2383  templateFootprints = None
2384 
2385  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2386  warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2387  if warpDiffExp is not None:
2388  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2389  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2390  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2391  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2392  fpSet.positive.merge(fpSet.negative)
2393  footprints = fpSet.positive
2394  slateIm.set(0)
2395  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2396 
2397  # Remove artifacts due to defects before they contribute to the epochCountImage
2398  if self.config.doPrefilterArtifacts:
2399  spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2400 
2401  # Clear mask before adding prefiltered spanSets
2402  self.detect.clearMask(warpDiffExp.mask)
2403  for spans in spanSetList:
2404  spans.setImage(slateIm, 1, doClip=True)
2405  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2406  epochCountImage += slateIm
2407 
2408  if self.config.doFilterMorphological:
2409  maskName = self.config.streakMaskName
2410  _ = self.maskStreaks.run(warpDiffExp)
2411  streakMask = warpDiffExp.mask
2412  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2413  streakMask.getPlaneBitMask(maskName)).split()
2414 
2415  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2416  # undergo a second convolution. Pixels with data in the direct warp
2417  # but not in the PSF-matched warp will not have their artifacts detected.
2418  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2419  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2420  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2421  nansMask.setXY0(warpDiffExp.getXY0())
2422  edgeMask = warpDiffExp.mask
2423  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2424  edgeMask.getPlaneBitMask("EDGE")).split()
2425  else:
2426  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2427  # In this case, mask the whole epoch
2428  nansMask = afwImage.MaskX(coaddBBox, 1)
2429  spanSetList = []
2430  spanSetEdgeMask = []
2431  spanSetStreak = []
2432 
2433  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2434 
2435  spanSetNoDataMaskList.append(spanSetNoDataMask)
2436  spanSetArtifactList.append(spanSetList)
2437  spanSetEdgeList.append(spanSetEdgeMask)
2438  if self.config.doFilterMorphological:
2439  spanSetBadMorphoList.append(spanSetStreak)
2440 
2441  if lsstDebug.Info(__name__).saveCountIm:
2442  path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2443  epochCountImage.writeFits(path)
2444 
2445  for i, spanSetList in enumerate(spanSetArtifactList):
2446  if spanSetList:
2447  filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2448  templateFootprints)
2449  spanSetArtifactList[i] = filteredSpanSetList
2450  if self.config.doFilterMorphological:
2451  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2452 
2453  altMasks = []
2454  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2455  altMasks.append({'CLIPPED': artifacts,
2456  'NO_DATA': noData,
2457  'EDGE': edge})
2458  return altMasks
2459 
2460  def prefilterArtifacts(self, spanSetList, exp):
2461  """Remove artifact candidates covered by bad mask plane.
2462 
2463  Any future editing of the candidate list that does not depend on
2464  temporal information should go in this method.
2465 
2466  Parameters
2467  ----------
2468  spanSetList : `list`
2469  List of SpanSets representing artifact candidates.
2470  exp : `lsst.afw.image.Exposure`
2471  Exposure containing mask planes used to prefilter.
2472 
2473  Returns
2474  -------
2475  returnSpanSetList : `list`
2476  List of SpanSets with artifacts.
2477  """
2478  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2479  goodArr = (exp.mask.array & badPixelMask) == 0
2480  returnSpanSetList = []
2481  bbox = exp.getBBox()
2482  x0, y0 = exp.getXY0()
2483  for i, span in enumerate(spanSetList):
2484  y, x = span.clippedTo(bbox).indices()
2485  yIndexLocal = numpy.array(y) - y0
2486  xIndexLocal = numpy.array(x) - x0
2487  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2488  if goodRatio > self.config.prefilterArtifactsRatio:
2489  returnSpanSetList.append(span)
2490  return returnSpanSetList
2491 
2492  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2493  """Filter artifact candidates.
2494 
2495  Parameters
2496  ----------
2497  spanSetList : `list`
2498  List of SpanSets representing artifact candidates.
2499  epochCountImage : `lsst.afw.image.Image`
2500  Image of accumulated number of warpDiff detections.
2501  nImage : `lsst.afw.image.Image`
2502  Image of the accumulated number of total epochs contributing.
2503 
2504  Returns
2505  -------
2506  maskSpanSetList : `list`
2507  List of SpanSets with artifacts.
2508  """
2509 
2510  maskSpanSetList = []
2511  x0, y0 = epochCountImage.getXY0()
2512  for i, span in enumerate(spanSetList):
2513  y, x = span.indices()
2514  yIdxLocal = [y1 - y0 for y1 in y]
2515  xIdxLocal = [x1 - x0 for x1 in x]
2516  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2517  totalN = nImage.array[yIdxLocal, xIdxLocal]
2518 
2519  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2520  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2521  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2522  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2523  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2524  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2525  & (outlierN <= effectiveMaxNumEpochs))
2526  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2527  if percentBelowThreshold > self.config.spatialThreshold:
2528  maskSpanSetList.append(span)
2529 
2530  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2531  # If a candidate is contained by a footprint on the template coadd, do not clip
2532  filteredMaskSpanSetList = []
2533  for span in maskSpanSetList:
2534  doKeep = True
2535  for footprint in footprintsToExclude.positive.getFootprints():
2536  if footprint.spans.contains(span):
2537  doKeep = False
2538  break
2539  if doKeep:
2540  filteredMaskSpanSetList.append(span)
2541  maskSpanSetList = filteredMaskSpanSetList
2542 
2543  return maskSpanSetList
2544 
2545  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2546  """Fetch a warp from the butler and return a warpDiff.
2547 
2548  Parameters
2549  ----------
2550  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2551  Butler dataRef for the warp.
2552  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2553  An image scaler object.
2554  templateCoadd : `lsst.afw.image.Exposure`
2555  Exposure to be substracted from the scaled warp.
2556 
2557  Returns
2558  -------
2559  warp : `lsst.afw.image.Exposure`
2560  Exposure of the image difference between the warp and template.
2561  """
2562 
2563  # If the PSF-Matched warp did not exist for this direct warp
2564  # None is holding its place to maintain order in Gen 3
2565  if warpRef is None:
2566  return None
2567  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2568  warpName = self.getTempExpDatasetName('psfMatched')
2569  if not isinstance(warpRef, DeferredDatasetHandle):
2570  if not warpRef.datasetExists(warpName):
2571  self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2572  return None
2573  warp = warpRef.get(datasetType=warpName, immediate=True)
2574  # direct image scaler OK for PSF-matched Warp
2575  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2576  mi = warp.getMaskedImage()
2577  if self.config.doScaleWarpVariance:
2578  try:
2579  self.scaleWarpVariance.run(mi)
2580  except Exception as exc:
2581  self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc)
2582  mi -= templateCoadd.getMaskedImage()
2583  return warp
2584 
2585  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2586  """Return a path to which to write debugging output.
2587 
2588  Creates a hyphen-delimited string of dataId values for simple filenames.
2589 
2590  Parameters
2591  ----------
2592  prefix : `str`
2593  Prefix for filename.
2594  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2595  Butler dataRef to make the path from.
2596  coaddLevel : `bool`, optional.
2597  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2598  'filter', but no 'visit').
2599 
2600  Returns
2601  -------
2602  result : `str`
2603  Path for debugging output.
2604  """
2605  if coaddLevel:
2606  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2607  else:
2608  keys = warpRef.dataId.keys()
2609  keyList = sorted(keys, reverse=True)
2610  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2611  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2612  return os.path.join(directory, filename)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:141
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:346
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60