lsst.pipe.tasks  21.0.0-20-g55224fe4+fb58a89680
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from lsst.meas.algorithms import SourceDetectionTask
48 from lsst.daf.butler import DeferredDatasetHandle
49 
50 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
51  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
52  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
53 
54 
55 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
56  dimensions=("tract", "patch", "band", "skymap"),
57  defaultTemplates={"inputCoaddName": "deep",
58  "outputCoaddName": "deep",
59  "warpType": "direct",
60  "warpTypeSuffix": "",
61  "fakesType": ""}):
62  inputWarps = pipeBase.connectionTypes.Input(
63  doc=("Input list of warps to be assemebled i.e. stacked."
64  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
65  name="{inputCoaddName}Coadd_{warpType}Warp",
66  storageClass="ExposureF",
67  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
68  deferLoad=True,
69  multiple=True
70  )
71  skyMap = pipeBase.connectionTypes.Input(
72  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
73  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
74  storageClass="SkyMap",
75  dimensions=("skymap", ),
76  )
77  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
78  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
79  " BRIGHT_OBJECT."),
80  name="brightObjectMask",
81  storageClass="ObjectMaskCatalog",
82  dimensions=("tract", "patch", "skymap", "band"),
83  )
84  coaddExposure = pipeBase.connectionTypes.Output(
85  doc="Output coadded exposure, produced by stacking input warps",
86  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
87  storageClass="ExposureF",
88  dimensions=("tract", "patch", "skymap", "band"),
89  )
90  nImage = pipeBase.connectionTypes.Output(
91  doc="Output image of number of input images per pixel",
92  name="{outputCoaddName}Coadd_nImage",
93  storageClass="ImageU",
94  dimensions=("tract", "patch", "skymap", "band"),
95  )
96 
97  def __init__(self, *, config=None):
98  super().__init__(config=config)
99 
100  # Override the connection's name template with config to replicate Gen2 behavior
101  # This duplicates some of the logic in the base class, due to wanting Gen2 and
102  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
103  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
104  templateValues['warpType'] = config.warpType
105  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
106  if config.hasFakes:
107  templateValues['fakesType'] = "_fakes"
108  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
109  for name in self.allConnections}
110  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
111  # End code to remove after deprecation
112 
113  if not config.doMaskBrightObjects:
114  self.prerequisiteInputs.remove("brightObjectMask")
115 
116  if not config.doNImage:
117  self.outputs.remove("nImage")
118 
119 
120 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
121  pipelineConnections=AssembleCoaddConnections):
122  """Configuration parameters for the `AssembleCoaddTask`.
123 
124  Notes
125  -----
126  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
127  only set the bitplane config.brightObjectMaskName. To make this useful you
128  *must* also configure the flags.pixel algorithm, for example by adding
129 
130  .. code-block:: none
131 
132  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
133  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
134 
135  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
136  """
137  warpType = pexConfig.Field(
138  doc="Warp name: one of 'direct' or 'psfMatched'",
139  dtype=str,
140  default="direct",
141  )
142  subregionSize = pexConfig.ListField(
143  dtype=int,
144  doc="Width, height of stack subregion size; "
145  "make small enough that a full stack of images will fit into memory at once.",
146  length=2,
147  default=(2000, 2000),
148  )
149  statistic = pexConfig.Field(
150  dtype=str,
151  doc="Main stacking statistic for aggregating over the epochs.",
152  default="MEANCLIP",
153  )
154  doSigmaClip = pexConfig.Field(
155  dtype=bool,
156  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
157  default=False,
158  )
159  sigmaClip = pexConfig.Field(
160  dtype=float,
161  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
162  default=3.0,
163  )
164  clipIter = pexConfig.Field(
165  dtype=int,
166  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
167  default=2,
168  )
169  calcErrorFromInputVariance = pexConfig.Field(
170  dtype=bool,
171  doc="Calculate coadd variance from input variance by stacking statistic."
172  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
173  default=True,
174  )
175  scaleZeroPoint = pexConfig.ConfigurableField(
176  target=ScaleZeroPointTask,
177  doc="Task to adjust the photometric zero point of the coadd temp exposures",
178  )
179  doInterp = pexConfig.Field(
180  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
181  dtype=bool,
182  default=True,
183  )
184  interpImage = pexConfig.ConfigurableField(
185  target=InterpImageTask,
186  doc="Task to interpolate (and extrapolate) over NaN pixels",
187  )
188  doWrite = pexConfig.Field(
189  doc="Persist coadd?",
190  dtype=bool,
191  default=True,
192  )
193  doNImage = pexConfig.Field(
194  doc="Create image of number of contributing exposures for each pixel",
195  dtype=bool,
196  default=False,
197  )
198  doUsePsfMatchedPolygons = pexConfig.Field(
199  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
200  dtype=bool,
201  default=False,
202  )
203  maskPropagationThresholds = pexConfig.DictField(
204  keytype=str,
205  itemtype=float,
206  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
207  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
208  "would have contributed exceeds this value."),
209  default={"SAT": 0.1},
210  )
211  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
212  doc="Mask planes to remove before coadding")
213  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
214  doc="Set mask and flag bits for bright objects?")
215  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
216  doc="Name of mask bit used for bright objects")
217  coaddPsf = pexConfig.ConfigField(
218  doc="Configuration for CoaddPsf",
219  dtype=measAlg.CoaddPsfConfig,
220  )
221  doAttachTransmissionCurve = pexConfig.Field(
222  dtype=bool, default=False, optional=False,
223  doc=("Attach a piecewise TransmissionCurve for the coadd? "
224  "(requires all input Exposures to have TransmissionCurves).")
225  )
226  hasFakes = pexConfig.Field(
227  dtype=bool,
228  default=False,
229  doc="Should be set to True if fake sources have been inserted into the input data."
230  )
231 
232  def setDefaults(self):
233  super().setDefaults()
234  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
235 
236  def validate(self):
237  super().validate()
238  if self.doPsfMatch:
239  # Backwards compatibility.
240  # Configs do not have loggers
241  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
242  self.warpType = 'psfMatched'
243  if self.doSigmaClip and self.statistic != "MEANCLIP":
244  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
245  self.statistic = "MEANCLIP"
246  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
247  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
248  "compute and set a non-zero coadd variance estimate." % (self.statistic))
249 
250  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
251  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
252  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
253  if str(k) not in unstackableStats]
254  raise ValueError("statistic %s is not allowed. Please choose one of %s."
255  % (self.statistic, stackableStats))
256 
257 
258 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
259  """Assemble a coadded image from a set of warps (coadded temporary exposures).
260 
261  We want to assemble a coadded image from a set of Warps (also called
262  coadded temporary exposures or ``coaddTempExps``).
263  Each input Warp covers a patch on the sky and corresponds to a single
264  run/visit/exposure of the covered patch. We provide the task with a list
265  of Warps (``selectDataList``) from which it selects Warps that cover the
266  specified patch (pointed at by ``dataRef``).
267  Each Warp that goes into a coadd will typically have an independent
268  photometric zero-point. Therefore, we must scale each Warp to set it to
269  a common photometric zeropoint. WarpType may be one of 'direct' or
270  'psfMatched', and the boolean configs `config.makeDirect` and
271  `config.makePsfMatched` set which of the warp types will be coadded.
272  The coadd is computed as a mean with optional outlier rejection.
273  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
274  Finally, Warps can have bad 'NaN' pixels which received no input from the
275  source calExps. We interpolate over these bad (NaN) pixels.
276 
277  `AssembleCoaddTask` uses several sub-tasks. These are
278 
279  - `ScaleZeroPointTask`
280  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
281  - `InterpImageTask`
282  - interpolate across bad pixels (NaN) in the final coadd
283 
284  You can retarget these subtasks if you wish.
285 
286  Notes
287  -----
288  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
289  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
290  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
291  no debug variables of its own. Some of the subtasks may support debug
292  variables. See the documentation for the subtasks for further information.
293 
294  Examples
295  --------
296  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
297  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
298  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
299  inputs: a data reference to the tract patch and filter to be coadded, and
300  a list of Warps to attempt to coadd. These are specified using ``--id`` and
301  ``--selectId``, respectively:
302 
303  .. code-block:: none
304 
305  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
306  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
307 
308  Only the Warps that cover the specified tract and patch will be coadded.
309  A list of the available optional arguments can be obtained by calling
310  ``assembleCoadd.py`` with the ``--help`` command line argument:
311 
312  .. code-block:: none
313 
314  assembleCoadd.py --help
315 
316  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
317  multi-band processing, we will generate the HSC-I & -R band coadds from
318  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
319  assuming that the lsst stack has been already set up, we must set up the
320  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
321  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
322  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
323  coadds, we must first
324 
325  - processCcd
326  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
327  - makeSkyMap
328  - create a skymap that covers the area of the sky present in the raw exposures
329  - makeCoaddTempExp
330  - warp the individual calibrated exposures to the tangent plane of the coadd
331 
332  We can perform all of these steps by running
333 
334  .. code-block:: none
335 
336  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
337 
338  This will produce warped exposures for each visit. To coadd the warped
339  data, we call assembleCoadd.py as follows:
340 
341  .. code-block:: none
342 
343  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
344  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
345  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
346  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
347  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
348  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
349  --selectId visit=903988 ccd=24
350 
351  that will process the HSC-I band data. The results are written in
352  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
353 
354  You may also choose to run:
355 
356  .. code-block:: none
357 
358  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
359  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
360  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
361  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
362  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
363  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
364  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
365  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
366 
367  to generate the coadd for the HSC-R band if you are interested in
368  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
369  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
370  rather than `AssembleCoaddTask` to make the coadd.
371  """
372  ConfigClass = AssembleCoaddConfig
373  _DefaultName = "assembleCoadd"
374 
375  def __init__(self, *args, **kwargs):
376  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
377  if args:
378  argNames = ["config", "name", "parentTask", "log"]
379  kwargs.update({k: v for k, v in zip(argNames, args)})
380  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
381  "PipelineTask will not take positional args" % argNames, FutureWarning)
382 
383  super().__init__(**kwargs)
384  self.makeSubtask("interpImage")
385  self.makeSubtask("scaleZeroPoint")
386 
387  if self.config.doMaskBrightObjects:
388  mask = afwImage.Mask()
389  try:
390  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
391  except pexExceptions.LsstCppException:
392  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
393  mask.getMaskPlaneDict().keys())
394  del mask
395 
396  self.warpType = self.config.warpType
397 
398  @utils.inheritDoc(pipeBase.PipelineTask)
399  def runQuantum(self, butlerQC, inputRefs, outputRefs):
400  # Docstring to be formatted with info from PipelineTask.runQuantum
401  """
402  Notes
403  -----
404  Assemble a coadd from a set of Warps.
405 
406  PipelineTask (Gen3) entry point to Coadd a set of Warps.
407  Analogous to `runDataRef`, it prepares all the data products to be
408  passed to `run`, and processes the results before returning a struct
409  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
410  Therefore, its inputs are accessed subregion by subregion
411  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
412  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
413  correspond to an update in `runDataRef` while both entry points
414  are used.
415  """
416  inputData = butlerQC.get(inputRefs)
417 
418  # Construct skyInfo expected by run
419  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
420  skyMap = inputData["skyMap"]
421  outputDataId = butlerQC.quantum.dataId
422 
423  inputData['skyInfo'] = makeSkyInfo(skyMap,
424  tractId=outputDataId['tract'],
425  patchId=outputDataId['patch'])
426 
427  # Construct list of input Deferred Datasets
428  # These quack a bit like like Gen2 DataRefs
429  warpRefList = inputData['inputWarps']
430  # Perform same middle steps as `runDataRef` does
431  inputs = self.prepareInputs(warpRefList)
432  self.log.info("Found %d %s", len(inputs.tempExpRefList),
433  self.getTempExpDatasetName(self.warpType))
434  if len(inputs.tempExpRefList) == 0:
435  self.log.warn("No coadd temporary exposures found")
436  return
437 
438  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
439  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
440  inputs.weightList, supplementaryData=supplementaryData)
441 
442  inputData.setdefault('brightObjectMask', None)
443  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
444 
445  if self.config.doWrite:
446  butlerQC.put(retStruct, outputRefs)
447  return retStruct
448 
449  @pipeBase.timeMethod
450  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
451  """Assemble a coadd from a set of Warps.
452 
453  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
454  Compute weights to be applied to each Warp and
455  find scalings to match the photometric zeropoint to a reference Warp.
456  Assemble the Warps using `run`. Interpolate over NaNs and
457  optionally write the coadd to disk. Return the coadded exposure.
458 
459  Parameters
460  ----------
461  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
462  Data reference defining the patch for coaddition and the
463  reference Warp (if ``config.autoReference=False``).
464  Used to access the following data products:
465  - ``self.config.coaddName + "Coadd_skyMap"``
466  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
467  - ``self.config.coaddName + "Coadd"``
468  selectDataList : `list`
469  List of data references to Calexps. Data to be coadded will be
470  selected from this list based on overlap with the patch defined
471  by dataRef, grouped by visit, and converted to a list of data
472  references to warps.
473  warpRefList : `list`
474  List of data references to Warps to be coadded.
475  Note: `warpRefList` is just the new name for `tempExpRefList`.
476 
477  Returns
478  -------
479  retStruct : `lsst.pipe.base.Struct`
480  Result struct with components:
481 
482  - ``coaddExposure``: coadded exposure (``Exposure``).
483  - ``nImage``: exposure count image (``Image``).
484  """
485  if selectDataList and warpRefList:
486  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
487  "and which to use is ambiguous. Please pass only one.")
488 
489  skyInfo = self.getSkyInfo(dataRef)
490  if warpRefList is None:
491  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
492  if len(calExpRefList) == 0:
493  self.log.warn("No exposures to coadd")
494  return
495  self.log.info("Coadding %d exposures", len(calExpRefList))
496 
497  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
498 
499  inputData = self.prepareInputs(warpRefList)
500  self.log.info("Found %d %s", len(inputData.tempExpRefList),
501  self.getTempExpDatasetName(self.warpType))
502  if len(inputData.tempExpRefList) == 0:
503  self.log.warn("No coadd temporary exposures found")
504  return
505 
506  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
507 
508  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
509  inputData.weightList, supplementaryData=supplementaryData)
510 
511  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
512  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
513 
514  if self.config.doWrite:
515  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
516  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
517  else:
518  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
519  self.log.info("Persisting %s" % coaddDatasetName)
520  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
521  if self.config.doNImage and retStruct.nImage is not None:
522  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
523 
524  return retStruct
525 
526  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
527  """Interpolate over missing data and mask bright stars.
528 
529  Parameters
530  ----------
531  coaddExposure : `lsst.afw.image.Exposure`
532  The coadded exposure to process.
533  dataRef : `lsst.daf.persistence.ButlerDataRef`
534  Butler data reference for supplementary data.
535  """
536  if self.config.doInterp:
537  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
538  # The variance must be positive; work around for DM-3201.
539  varArray = coaddExposure.variance.array
540  with numpy.errstate(invalid="ignore"):
541  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
542 
543  if self.config.doMaskBrightObjects:
544  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
545 
546  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
547  """Make additional inputs to run() specific to subclasses (Gen2)
548 
549  Duplicates interface of `runDataRef` method
550  Available to be implemented by subclasses only if they need the
551  coadd dataRef for performing preliminary processing before
552  assembling the coadd.
553 
554  Parameters
555  ----------
556  dataRef : `lsst.daf.persistence.ButlerDataRef`
557  Butler data reference for supplementary data.
558  selectDataList : `list` (optional)
559  Optional List of data references to Calexps.
560  warpRefList : `list` (optional)
561  Optional List of data references to Warps.
562  """
563  return pipeBase.Struct()
564 
565  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
566  """Make additional inputs to run() specific to subclasses (Gen3)
567 
568  Duplicates interface of `runQuantum` method.
569  Available to be implemented by subclasses only if they need the
570  coadd dataRef for performing preliminary processing before
571  assembling the coadd.
572 
573  Parameters
574  ----------
575  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
576  Gen3 Butler object for fetching additional data products before
577  running the Task specialized for quantum being processed
578  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
579  Attributes are the names of the connections describing input dataset types.
580  Values are DatasetRefs that task consumes for corresponding dataset type.
581  DataIds are guaranteed to match data objects in ``inputData``.
582  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
583  Attributes are the names of the connections describing output dataset types.
584  Values are DatasetRefs that task is to produce
585  for corresponding dataset type.
586  """
587  return pipeBase.Struct()
588 
589  def getTempExpRefList(self, patchRef, calExpRefList):
590  """Generate list data references corresponding to warped exposures
591  that lie within the patch to be coadded.
592 
593  Parameters
594  ----------
595  patchRef : `dataRef`
596  Data reference for patch.
597  calExpRefList : `list`
598  List of data references for input calexps.
599 
600  Returns
601  -------
602  tempExpRefList : `list`
603  List of Warp/CoaddTempExp data references.
604  """
605  butler = patchRef.getButler()
606  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
607  self.getTempExpDatasetName(self.warpType))
608  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
609  g, groupData.keys) for
610  g in groupData.groups.keys()]
611  return tempExpRefList
612 
613  def prepareInputs(self, refList):
614  """Prepare the input warps for coaddition by measuring the weight for
615  each warp and the scaling for the photometric zero point.
616 
617  Each Warp has its own photometric zeropoint and background variance.
618  Before coadding these Warps together, compute a scale factor to
619  normalize the photometric zeropoint and compute the weight for each Warp.
620 
621  Parameters
622  ----------
623  refList : `list`
624  List of data references to tempExp
625 
626  Returns
627  -------
628  result : `lsst.pipe.base.Struct`
629  Result struct with components:
630 
631  - ``tempExprefList``: `list` of data references to tempExp.
632  - ``weightList``: `list` of weightings.
633  - ``imageScalerList``: `list` of image scalers.
634  """
635  statsCtrl = afwMath.StatisticsControl()
636  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
637  statsCtrl.setNumIter(self.config.clipIter)
638  statsCtrl.setAndMask(self.getBadPixelMask())
639  statsCtrl.setNanSafe(True)
640  # compute tempExpRefList: a list of tempExpRef that actually exist
641  # and weightList: a list of the weight of the associated coadd tempExp
642  # and imageScalerList: a list of scale factors for the associated coadd tempExp
643  tempExpRefList = []
644  weightList = []
645  imageScalerList = []
646  tempExpName = self.getTempExpDatasetName(self.warpType)
647  for tempExpRef in refList:
648  # Gen3's DeferredDatasetHandles are guaranteed to exist and
649  # therefore have no datasetExists() method
650  if not isinstance(tempExpRef, DeferredDatasetHandle):
651  if not tempExpRef.datasetExists(tempExpName):
652  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
653  continue
654 
655  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
656  # Ignore any input warp that is empty of data
657  if numpy.isnan(tempExp.image.array).all():
658  continue
659  maskedImage = tempExp.getMaskedImage()
660  imageScaler = self.scaleZeroPoint.computeImageScaler(
661  exposure=tempExp,
662  dataRef=tempExpRef,
663  )
664  try:
665  imageScaler.scaleMaskedImage(maskedImage)
666  except Exception as e:
667  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
668  continue
669  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
670  afwMath.MEANCLIP, statsCtrl)
671  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
672  weight = 1.0 / float(meanVar)
673  if not numpy.isfinite(weight):
674  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
675  continue
676  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
677 
678  del maskedImage
679  del tempExp
680 
681  tempExpRefList.append(tempExpRef)
682  weightList.append(weight)
683  imageScalerList.append(imageScaler)
684 
685  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
686  imageScalerList=imageScalerList)
687 
688  def prepareStats(self, mask=None):
689  """Prepare the statistics for coadding images.
690 
691  Parameters
692  ----------
693  mask : `int`, optional
694  Bit mask value to exclude from coaddition.
695 
696  Returns
697  -------
698  stats : `lsst.pipe.base.Struct`
699  Statistics structure with the following fields:
700 
701  - ``statsCtrl``: Statistics control object for coadd
702  (`lsst.afw.math.StatisticsControl`)
703  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
704  """
705  if mask is None:
706  mask = self.getBadPixelMask()
707  statsCtrl = afwMath.StatisticsControl()
708  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
709  statsCtrl.setNumIter(self.config.clipIter)
710  statsCtrl.setAndMask(mask)
711  statsCtrl.setNanSafe(True)
712  statsCtrl.setWeighted(True)
713  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
714  for plane, threshold in self.config.maskPropagationThresholds.items():
715  bit = afwImage.Mask.getMaskPlane(plane)
716  statsCtrl.setMaskPropagationThreshold(bit, threshold)
717  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
718  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
719 
720  @pipeBase.timeMethod
721  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
722  altMaskList=None, mask=None, supplementaryData=None):
723  """Assemble a coadd from input warps
724 
725  Assemble the coadd using the provided list of coaddTempExps. Since
726  the full coadd covers a patch (a large area), the assembly is
727  performed over small areas on the image at a time in order to
728  conserve memory usage. Iterate over subregions within the outer
729  bbox of the patch using `assembleSubregion` to stack the corresponding
730  subregions from the coaddTempExps with the statistic specified.
731  Set the edge bits the coadd mask based on the weight map.
732 
733  Parameters
734  ----------
735  skyInfo : `lsst.pipe.base.Struct`
736  Struct with geometric information about the patch.
737  tempExpRefList : `list`
738  List of data references to Warps (previously called CoaddTempExps).
739  imageScalerList : `list`
740  List of image scalers.
741  weightList : `list`
742  List of weights
743  altMaskList : `list`, optional
744  List of alternate masks to use rather than those stored with
745  tempExp.
746  mask : `int`, optional
747  Bit mask value to exclude from coaddition.
748  supplementaryData : lsst.pipe.base.Struct, optional
749  Struct with additional data products needed to assemble coadd.
750  Only used by subclasses that implement `makeSupplementaryData`
751  and override `run`.
752 
753  Returns
754  -------
755  result : `lsst.pipe.base.Struct`
756  Result struct with components:
757 
758  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
759  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
760  - ``warpRefList``: input list of refs to the warps (
761  ``lsst.daf.butler.DeferredDatasetHandle`` or
762  ``lsst.daf.persistence.ButlerDataRef``)
763  (unmodified)
764  - ``imageScalerList``: input list of image scalers (unmodified)
765  - ``weightList``: input list of weights (unmodified)
766  """
767  tempExpName = self.getTempExpDatasetName(self.warpType)
768  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
769  stats = self.prepareStats(mask=mask)
770 
771  if altMaskList is None:
772  altMaskList = [None]*len(tempExpRefList)
773 
774  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
775  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
776  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
777  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
778  coaddMaskedImage = coaddExposure.getMaskedImage()
779  subregionSizeArr = self.config.subregionSize
780  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
781  # if nImage is requested, create a zero one which can be passed to assembleSubregion
782  if self.config.doNImage:
783  nImage = afwImage.ImageU(skyInfo.bbox)
784  else:
785  nImage = None
786  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
787  try:
788  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
789  weightList, altMaskList, stats.flags, stats.ctrl,
790  nImage=nImage)
791  except Exception as e:
792  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
793 
794  self.setInexactPsf(coaddMaskedImage.getMask())
795  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
796  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
797  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
798  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
799  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
800  weightList=weightList)
801 
802  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
803  """Set the metadata for the coadd.
804 
805  This basic implementation sets the filter from the first input.
806 
807  Parameters
808  ----------
809  coaddExposure : `lsst.afw.image.Exposure`
810  The target exposure for the coadd.
811  tempExpRefList : `list`
812  List of data references to tempExp.
813  weightList : `list`
814  List of weights.
815  """
816  assert len(tempExpRefList) == len(weightList), "Length mismatch"
817  tempExpName = self.getTempExpDatasetName(self.warpType)
818  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
819  # (and we need more than just the PropertySet that contains the header), which is not possible
820  # with the current butler (see #2777).
821  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
822 
823  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
824  # Gen 3 API
825  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
826  else:
827  # Gen 2 API. Delete this when Gen 2 retired
828  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
829  for tempExpRef in tempExpRefList]
830  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
831 
832  coaddExposure.setFilter(tempExpList[0].getFilter())
833  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
834  coaddInputs.ccds.reserve(numCcds)
835  coaddInputs.visits.reserve(len(tempExpList))
836 
837  for tempExp, weight in zip(tempExpList, weightList):
838  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
839 
840  if self.config.doUsePsfMatchedPolygons:
841  self.shrinkValidPolygons(coaddInputs)
842 
843  coaddInputs.visits.sort()
844  if self.warpType == "psfMatched":
845  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
846  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
847  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
848  # having the maximum width (sufficient because square)
849  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
850  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
851  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
852  else:
853  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
854  self.config.coaddPsf.makeControl())
855  coaddExposure.setPsf(psf)
856  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
857  coaddExposure.getWcs())
858  coaddExposure.getInfo().setApCorrMap(apCorrMap)
859  if self.config.doAttachTransmissionCurve:
860  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
861  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
862 
863  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
864  altMaskList, statsFlags, statsCtrl, nImage=None):
865  """Assemble the coadd for a sub-region.
866 
867  For each coaddTempExp, check for (and swap in) an alternative mask
868  if one is passed. Remove mask planes listed in
869  `config.removeMaskPlanes`. Finally, stack the actual exposures using
870  `lsst.afw.math.statisticsStack` with the statistic specified by
871  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
872  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
873  an N-sigma clipped mean where N and iterations are specified by
874  statsCtrl. Assign the stacked subregion back to the coadd.
875 
876  Parameters
877  ----------
878  coaddExposure : `lsst.afw.image.Exposure`
879  The target exposure for the coadd.
880  bbox : `lsst.geom.Box`
881  Sub-region to coadd.
882  tempExpRefList : `list`
883  List of data reference to tempExp.
884  imageScalerList : `list`
885  List of image scalers.
886  weightList : `list`
887  List of weights.
888  altMaskList : `list`
889  List of alternate masks to use rather than those stored with
890  tempExp, or None. Each element is dict with keys = mask plane
891  name to which to add the spans.
892  statsFlags : `lsst.afw.math.Property`
893  Property object for statistic for coadd.
894  statsCtrl : `lsst.afw.math.StatisticsControl`
895  Statistics control object for coadd.
896  nImage : `lsst.afw.image.ImageU`, optional
897  Keeps track of exposure count for each pixel.
898  """
899  self.log.debug("Computing coadd over %s", bbox)
900  tempExpName = self.getTempExpDatasetName(self.warpType)
901  coaddExposure.mask.addMaskPlane("REJECTED")
902  coaddExposure.mask.addMaskPlane("CLIPPED")
903  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
904  maskMap = self.setRejectedMaskMapping(statsCtrl)
905  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
906  maskedImageList = []
907  if nImage is not None:
908  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
909  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
910 
911  if isinstance(tempExpRef, DeferredDatasetHandle):
912  # Gen 3 API
913  exposure = tempExpRef.get(parameters={'bbox': bbox})
914  else:
915  # Gen 2 API. Delete this when Gen 2 retired
916  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
917 
918  maskedImage = exposure.getMaskedImage()
919  mask = maskedImage.getMask()
920  if altMask is not None:
921  self.applyAltMaskPlanes(mask, altMask)
922  imageScaler.scaleMaskedImage(maskedImage)
923 
924  # Add 1 for each pixel which is not excluded by the exclude mask.
925  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
926  if nImage is not None:
927  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
928  if self.config.removeMaskPlanes:
929  self.removeMaskPlanes(maskedImage)
930  maskedImageList.append(maskedImage)
931 
932  with self.timer("stack"):
933  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
934  clipped, # also set output to CLIPPED if sigma-clipped
935  maskMap)
936  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
937  if nImage is not None:
938  nImage.assign(subNImage, bbox)
939 
940  def removeMaskPlanes(self, maskedImage):
941  """Unset the mask of an image for mask planes specified in the config.
942 
943  Parameters
944  ----------
945  maskedImage : `lsst.afw.image.MaskedImage`
946  The masked image to be modified.
947  """
948  mask = maskedImage.getMask()
949  for maskPlane in self.config.removeMaskPlanes:
950  try:
951  mask &= ~mask.getPlaneBitMask(maskPlane)
952  except pexExceptions.InvalidParameterError:
953  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
954  maskPlane)
955 
956  @staticmethod
957  def setRejectedMaskMapping(statsCtrl):
958  """Map certain mask planes of the warps to new planes for the coadd.
959 
960  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
961  or CLIPPED, set it to REJECTED on the coadd.
962  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
963  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
964 
965  Parameters
966  ----------
967  statsCtrl : `lsst.afw.math.StatisticsControl`
968  Statistics control object for coadd
969 
970  Returns
971  -------
972  maskMap : `list` of `tuple` of `int`
973  A list of mappings of mask planes of the warped exposures to
974  mask planes of the coadd.
975  """
976  edge = afwImage.Mask.getPlaneBitMask("EDGE")
977  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
978  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
979  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
980  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
981  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
982  (clipped, clipped)]
983  return maskMap
984 
985  def applyAltMaskPlanes(self, mask, altMaskSpans):
986  """Apply in place alt mask formatted as SpanSets to a mask.
987 
988  Parameters
989  ----------
990  mask : `lsst.afw.image.Mask`
991  Original mask.
992  altMaskSpans : `dict`
993  SpanSet lists to apply. Each element contains the new mask
994  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
995  and list of SpanSets to apply to the mask.
996 
997  Returns
998  -------
999  mask : `lsst.afw.image.Mask`
1000  Updated mask.
1001  """
1002  if self.config.doUsePsfMatchedPolygons:
1003  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1004  # Clear away any other masks outside the validPolygons. These pixels are no longer
1005  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1006  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1007  # This mask-clearing step must only occur *before* applying the new masks below
1008  for spanSet in altMaskSpans['NO_DATA']:
1009  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1010 
1011  for plane, spanSetList in altMaskSpans.items():
1012  maskClipValue = mask.addMaskPlane(plane)
1013  for spanSet in spanSetList:
1014  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1015  return mask
1016 
1017  def shrinkValidPolygons(self, coaddInputs):
1018  """Shrink coaddInputs' ccds' ValidPolygons in place.
1019 
1020  Either modify each ccd's validPolygon in place, or if CoaddInputs
1021  does not have a validPolygon, create one from its bbox.
1022 
1023  Parameters
1024  ----------
1025  coaddInputs : `lsst.afw.image.coaddInputs`
1026  Original mask.
1027 
1028  """
1029  for ccd in coaddInputs.ccds:
1030  polyOrig = ccd.getValidPolygon()
1031  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1032  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1033  if polyOrig:
1034  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1035  else:
1036  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1037  ccd.setValidPolygon(validPolygon)
1038 
1039  def readBrightObjectMasks(self, dataRef):
1040  """Retrieve the bright object masks.
1041 
1042  Returns None on failure.
1043 
1044  Parameters
1045  ----------
1046  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1047  A Butler dataRef.
1048 
1049  Returns
1050  -------
1051  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1052  Bright object mask from the Butler object, or None if it cannot
1053  be retrieved.
1054  """
1055  try:
1056  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1057  except Exception as e:
1058  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1059  return None
1060 
1061  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1062  """Set the bright object masks.
1063 
1064  Parameters
1065  ----------
1066  exposure : `lsst.afw.image.Exposure`
1067  Exposure under consideration.
1068  dataId : `lsst.daf.persistence.dataId`
1069  Data identifier dict for patch.
1070  brightObjectMasks : `lsst.afw.table`
1071  Table of bright objects to mask.
1072  """
1073 
1074  if brightObjectMasks is None:
1075  self.log.warn("Unable to apply bright object mask: none supplied")
1076  return
1077  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1078  mask = exposure.getMaskedImage().getMask()
1079  wcs = exposure.getWcs()
1080  plateScale = wcs.getPixelScale().asArcseconds()
1081 
1082  for rec in brightObjectMasks:
1083  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1084  if rec["type"] == "box":
1085  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1086  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1087  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1088 
1089  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1090  bbox = geom.Box2I(center - halfSize, center + halfSize)
1091 
1092  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1093  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1094  spans = afwGeom.SpanSet(bbox)
1095  elif rec["type"] == "circle":
1096  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1097  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1098  else:
1099  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1100  continue
1101  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1102 
1103  def setInexactPsf(self, mask):
1104  """Set INEXACT_PSF mask plane.
1105 
1106  If any of the input images isn't represented in the coadd (due to
1107  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1108  these pixels.
1109 
1110  Parameters
1111  ----------
1112  mask : `lsst.afw.image.Mask`
1113  Coadded exposure's mask, modified in-place.
1114  """
1115  mask.addMaskPlane("INEXACT_PSF")
1116  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1117  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1118  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1119  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1120  array = mask.getArray()
1121  selected = array & (sensorEdge | clipped | rejected) > 0
1122  array[selected] |= inexactPsf
1123 
1124  @classmethod
1125  def _makeArgumentParser(cls):
1126  """Create an argument parser.
1127  """
1128  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1129  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1130  + cls.ConfigClass().warpType + "Warp",
1131  help="data ID, e.g. --id tract=12345 patch=1,2",
1132  ContainerClass=AssembleCoaddDataIdContainer)
1133  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1134  ContainerClass=SelectDataIdContainer)
1135  return parser
1136 
1137  @staticmethod
1138  def _subBBoxIter(bbox, subregionSize):
1139  """Iterate over subregions of a bbox.
1140 
1141  Parameters
1142  ----------
1143  bbox : `lsst.geom.Box2I`
1144  Bounding box over which to iterate.
1145  subregionSize: `lsst.geom.Extent2I`
1146  Size of sub-bboxes.
1147 
1148  Yields
1149  ------
1150  subBBox : `lsst.geom.Box2I`
1151  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1152  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1153  the edges of ``bbox``, but it will never be empty.
1154  """
1155  if bbox.isEmpty():
1156  raise RuntimeError("bbox %s is empty" % (bbox,))
1157  if subregionSize[0] < 1 or subregionSize[1] < 1:
1158  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1159 
1160  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1161  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1162  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1163  subBBox.clip(bbox)
1164  if subBBox.isEmpty():
1165  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1166  "colShift=%s, rowShift=%s" %
1167  (bbox, subregionSize, colShift, rowShift))
1168  yield subBBox
1169 
1170 
1171 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1172  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1173  """
1174 
1175  def makeDataRefList(self, namespace):
1176  """Make self.refList from self.idList.
1177 
1178  Parameters
1179  ----------
1180  namespace
1181  Results of parsing command-line (with ``butler`` and ``log`` elements).
1182  """
1183  datasetType = namespace.config.coaddName + "Coadd"
1184  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1185 
1186  for dataId in self.idList:
1187  # tract and patch are required
1188  for key in keysCoadd:
1189  if key not in dataId:
1190  raise RuntimeError("--id must include " + key)
1191 
1192  dataRef = namespace.butler.dataRef(
1193  datasetType=datasetType,
1194  dataId=dataId,
1195  )
1196  self.refList.append(dataRef)
1197 
1198 
1199 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1200  """Function to count the number of pixels with a specific mask in a
1201  footprint.
1202 
1203  Find the intersection of mask & footprint. Count all pixels in the mask
1204  that are in the intersection that have bitmask set but do not have
1205  ignoreMask set. Return the count.
1206 
1207  Parameters
1208  ----------
1209  mask : `lsst.afw.image.Mask`
1210  Mask to define intersection region by.
1211  footprint : `lsst.afw.detection.Footprint`
1212  Footprint to define the intersection region by.
1213  bitmask
1214  Specific mask that we wish to count the number of occurances of.
1215  ignoreMask
1216  Pixels to not consider.
1217 
1218  Returns
1219  -------
1220  result : `int`
1221  Count of number of pixels in footprint with specified mask.
1222  """
1223  bbox = footprint.getBBox()
1224  bbox.clip(mask.getBBox(afwImage.PARENT))
1225  fp = afwImage.Mask(bbox)
1226  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1227  footprint.spans.setMask(fp, bitmask)
1228  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1229  (subMask.getArray() & ignoreMask) == 0).sum()
1230 
1231 
1232 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1233  """Configuration parameters for the SafeClipAssembleCoaddTask.
1234  """
1235  assembleMeanCoadd = pexConfig.ConfigurableField(
1236  target=AssembleCoaddTask,
1237  doc="Task to assemble an initial Coadd using the MEAN statistic.",
1238  )
1239  assembleMeanClipCoadd = pexConfig.ConfigurableField(
1240  target=AssembleCoaddTask,
1241  doc="Task to assemble an initial Coadd using the MEANCLIP statistic.",
1242  )
1243  clipDetection = pexConfig.ConfigurableField(
1244  target=SourceDetectionTask,
1245  doc="Detect sources on difference between unclipped and clipped coadd")
1246  minClipFootOverlap = pexConfig.Field(
1247  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1248  dtype=float,
1249  default=0.6
1250  )
1251  minClipFootOverlapSingle = pexConfig.Field(
1252  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1253  "clipped when only one visit overlaps",
1254  dtype=float,
1255  default=0.5
1256  )
1257  minClipFootOverlapDouble = pexConfig.Field(
1258  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1259  "clipped when two visits overlap",
1260  dtype=float,
1261  default=0.45
1262  )
1263  maxClipFootOverlapDouble = pexConfig.Field(
1264  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1265  "considering two visits",
1266  dtype=float,
1267  default=0.15
1268  )
1269  minBigOverlap = pexConfig.Field(
1270  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1271  "when labeling clipped footprints",
1272  dtype=int,
1273  default=100
1274  )
1275 
1276  def setDefaults(self):
1277  """Set default values for clipDetection.
1278 
1279  Notes
1280  -----
1281  The numeric values for these configuration parameters were
1282  empirically determined, future work may further refine them.
1283  """
1284  AssembleCoaddConfig.setDefaults(self)
1285  self.clipDetection.doTempLocalBackground = False
1286  self.clipDetection.reEstimateBackground = False
1287  self.clipDetection.returnOriginalFootprints = False
1288  self.clipDetection.thresholdPolarity = "both"
1289  self.clipDetection.thresholdValue = 2
1290  self.clipDetection.nSigmaToGrow = 2
1291  self.clipDetection.minPixels = 4
1292  self.clipDetection.isotropicGrow = True
1293  self.clipDetection.thresholdType = "pixel_stdev"
1294  self.sigmaClip = 1.5
1295  self.clipIter = 3
1296  self.statistic = "MEAN"
1297  self.assembleMeanCoadd.statistic = 'MEAN'
1298  self.assembleMeanClipCoadd.statistic = 'MEANCLIP'
1299  self.assembleMeanCoadd.doWrite = False
1300  self.assembleMeanClipCoadd.doWrite = False
1301 
1302  def validate(self):
1303  if self.doSigmaClip:
1304  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1305  "Ignoring doSigmaClip.")
1306  self.doSigmaClip = False
1307  if self.statistic != "MEAN":
1308  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1309  "(%s chosen). Please set statistic to MEAN."
1310  % (self.statistic))
1311  AssembleCoaddTask.ConfigClass.validate(self)
1312 
1313 
1314 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1315  """Assemble a coadded image from a set of coadded temporary exposures,
1316  being careful to clip & flag areas with potential artifacts.
1317 
1318  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1319  we clip outliers). The problem with doing this is that when computing the
1320  coadd PSF at a given location, individual visit PSFs from visits with
1321  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1322  In this task, we correct for this behavior by creating a new
1323  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1324  coaddTempExps and the final coadd where
1325 
1326  i. difference imaging suggests that there is an outlier and
1327  ii. this outlier appears on only one or two images.
1328 
1329  Such regions will not contribute to the final coadd. Furthermore, any
1330  routine to determine the coadd PSF can now be cognizant of clipped regions.
1331  Note that the algorithm implemented by this task is preliminary and works
1332  correctly for HSC data. Parameter modifications and or considerable
1333  redesigning of the algorithm is likley required for other surveys.
1334 
1335  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1336  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1337  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1338  if you wish.
1339 
1340  Notes
1341  -----
1342  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1343  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1344  see `baseDebug` for more about ``debug.py`` files.
1345  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1346  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1347  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1348  for further information.
1349 
1350  Examples
1351  --------
1352  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1353  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1354  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1355 
1356  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1357  and filter to be coadded (specified using
1358  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1359  along with a list of coaddTempExps to attempt to coadd (specified using
1360  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1361  Only the coaddTempExps that cover the specified tract and patch will be
1362  coadded. A list of the available optional arguments can be obtained by
1363  calling assembleCoadd.py with the --help command line argument:
1364 
1365  .. code-block:: none
1366 
1367  assembleCoadd.py --help
1368 
1369  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1370  context of multi-band processing, we will generate the HSC-I & -R band
1371  coadds from HSC engineering test data provided in the ci_hsc package.
1372  To begin, assuming that the lsst stack has been already set up, we must
1373  set up the obs_subaru and ci_hsc packages. This defines the environment
1374  variable $CI_HSC_DIR and points at the location of the package. The raw
1375  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1376  the coadds, we must first
1377 
1378  - ``processCcd``
1379  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1380  - ``makeSkyMap``
1381  create a skymap that covers the area of the sky present in the raw exposures
1382  - ``makeCoaddTempExp``
1383  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1384 
1385  We can perform all of these steps by running
1386 
1387  .. code-block:: none
1388 
1389  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1390 
1391  This will produce warped coaddTempExps for each visit. To coadd the
1392  warped data, we call ``assembleCoadd.py`` as follows:
1393 
1394  .. code-block:: none
1395 
1396  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1397  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1398  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1399  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1400  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1401  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1402  --selectId visit=903988 ccd=24
1403 
1404  This will process the HSC-I band data. The results are written in
1405  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1406 
1407  You may also choose to run:
1408 
1409  .. code-block:: none
1410 
1411  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1412  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1413  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1414  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1415  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1416  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1417  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1418  --selectId visit=903346 ccd=12
1419 
1420  to generate the coadd for the HSC-R band if you are interested in following
1421  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1422  """
1423  ConfigClass = SafeClipAssembleCoaddConfig
1424  _DefaultName = "safeClipAssembleCoadd"
1425 
1426  def __init__(self, *args, **kwargs):
1427  AssembleCoaddTask.__init__(self, *args, **kwargs)
1428  schema = afwTable.SourceTable.makeMinimalSchema()
1429  self.makeSubtask("clipDetection", schema=schema)
1430  self.makeSubtask("assembleMeanClipCoadd")
1431  self.makeSubtask("assembleMeanCoadd")
1432 
1433  @utils.inheritDoc(AssembleCoaddTask)
1434  @pipeBase.timeMethod
1435  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1436  """Assemble the coadd for a region.
1437 
1438  Compute the difference of coadds created with and without outlier
1439  rejection to identify coadd pixels that have outlier values in some
1440  individual visits.
1441  Detect clipped regions on the difference image and mark these regions
1442  on the one or two individual coaddTempExps where they occur if there
1443  is significant overlap between the clipped region and a source. This
1444  leaves us with a set of footprints from the difference image that have
1445  been identified as having occured on just one or two individual visits.
1446  However, these footprints were generated from a difference image. It
1447  is conceivable for a large diffuse source to have become broken up
1448  into multiple footprints acrosss the coadd difference in this process.
1449  Determine the clipped region from all overlapping footprints from the
1450  detected sources in each visit - these are big footprints.
1451  Combine the small and big clipped footprints and mark them on a new
1452  bad mask plane.
1453  Generate the coadd using `AssembleCoaddTask.run` without outlier
1454  removal. Clipped footprints will no longer make it into the coadd
1455  because they are marked in the new bad mask plane.
1456 
1457  Notes
1458  -----
1459  args and kwargs are passed but ignored in order to match the call
1460  signature expected by the parent task.
1461  """
1462  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1463  mask = exp.getMaskedImage().getMask()
1464  mask.addMaskPlane("CLIPPED")
1465 
1466  result = self.detectClip(exp, tempExpRefList)
1467 
1468  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1469 
1470  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1471  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1472  # Append big footprints from individual Warps to result.clipSpans
1473  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1474  result.detectionFootprints, maskClipValue, maskDetValue,
1475  exp.getBBox())
1476  # Create mask of the current clipped footprints
1477  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1478  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1479 
1480  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1481  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1482  maskClip |= maskClipBig
1483 
1484  # Assemble coadd from base class, but ignoring CLIPPED pixels
1485  badMaskPlanes = self.config.badMaskPlanes[:]
1486  badMaskPlanes.append("CLIPPED")
1487  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1488  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1489  result.clipSpans, mask=badPixelMask)
1490 
1491  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1492  """Return an exposure that contains the difference between unclipped
1493  and clipped coadds.
1494 
1495  Generate a difference image between clipped and unclipped coadds.
1496  Compute the difference image by subtracting an outlier-clipped coadd
1497  from an outlier-unclipped coadd. Return the difference image.
1498 
1499  Parameters
1500  ----------
1501  skyInfo : `lsst.pipe.base.Struct`
1502  Patch geometry information, from getSkyInfo
1503  tempExpRefList : `list`
1504  List of data reference to tempExp
1505  imageScalerList : `list`
1506  List of image scalers
1507  weightList : `list`
1508  List of weights
1509 
1510  Returns
1511  -------
1512  exp : `lsst.afw.image.Exposure`
1513  Difference image of unclipped and clipped coadd wrapped in an Exposure
1514  """
1515  coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList,
1516  imageScalerList, weightList).coaddExposure
1517 
1518  coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList,
1519  imageScalerList, weightList).coaddExposure
1520 
1521  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1522  coaddDiff -= coaddClip.getMaskedImage()
1523  exp = afwImage.ExposureF(coaddDiff)
1524  exp.setPsf(coaddMean.getPsf())
1525  return exp
1526 
1527  def detectClip(self, exp, tempExpRefList):
1528  """Detect clipped regions on an exposure and set the mask on the
1529  individual tempExp masks.
1530 
1531  Detect footprints in the difference image after smoothing the
1532  difference image with a Gaussian kernal. Identify footprints that
1533  overlap with one or two input ``coaddTempExps`` by comparing the
1534  computed overlap fraction to thresholds set in the config. A different
1535  threshold is applied depending on the number of overlapping visits
1536  (restricted to one or two). If the overlap exceeds the thresholds,
1537  the footprint is considered "CLIPPED" and is marked as such on the
1538  coaddTempExp. Return a struct with the clipped footprints, the indices
1539  of the ``coaddTempExps`` that end up overlapping with the clipped
1540  footprints, and a list of new masks for the ``coaddTempExps``.
1541 
1542  Parameters
1543  ----------
1544  exp : `lsst.afw.image.Exposure`
1545  Exposure to run detection on.
1546  tempExpRefList : `list`
1547  List of data reference to tempExp.
1548 
1549  Returns
1550  -------
1551  result : `lsst.pipe.base.Struct`
1552  Result struct with components:
1553 
1554  - ``clipFootprints``: list of clipped footprints.
1555  - ``clipIndices``: indices for each ``clippedFootprint`` in
1556  ``tempExpRefList``.
1557  - ``clipSpans``: List of dictionaries containing spanSet lists
1558  to clip. Each element contains the new maskplane name
1559  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1560  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1561  compressed into footprints.
1562  """
1563  mask = exp.getMaskedImage().getMask()
1564  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1565  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1566  # Merge positive and negative together footprints together
1567  fpSet.positive.merge(fpSet.negative)
1568  footprints = fpSet.positive
1569  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1570  ignoreMask = self.getBadPixelMask()
1571 
1572  clipFootprints = []
1573  clipIndices = []
1574  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1575 
1576  # for use by detectClipBig
1577  visitDetectionFootprints = []
1578 
1579  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1580  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1581  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1582 
1583  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1584  for i, warpRef in enumerate(tempExpRefList):
1585  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1586  immediate=True).getMaskedImage().getMask()
1587  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1588  afwImage.PARENT, True)
1589  maskVisitDet &= maskDetValue
1590  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1591  visitDetectionFootprints.append(visitFootprints)
1592 
1593  for j, footprint in enumerate(footprints.getFootprints()):
1594  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1595  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1596 
1597  # build a list of clipped spans for each visit
1598  for j, footprint in enumerate(footprints.getFootprints()):
1599  nPixel = footprint.getArea()
1600  overlap = [] # hold the overlap with each visit
1601  indexList = [] # index of visit in global list
1602  for i in range(len(tempExpRefList)):
1603  ignore = ignoreArr[i, j]
1604  overlapDet = overlapDetArr[i, j]
1605  totPixel = nPixel - ignore
1606 
1607  # If we have more bad pixels than detection skip
1608  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1609  continue
1610  overlap.append(overlapDet/float(totPixel))
1611  indexList.append(i)
1612 
1613  overlap = numpy.array(overlap)
1614  if not len(overlap):
1615  continue
1616 
1617  keep = False # Should this footprint be marked as clipped?
1618  keepIndex = [] # Which tempExps does the clipped footprint belong to
1619 
1620  # If footprint only has one overlap use a lower threshold
1621  if len(overlap) == 1:
1622  if overlap[0] > self.config.minClipFootOverlapSingle:
1623  keep = True
1624  keepIndex = [0]
1625  else:
1626  # This is the general case where only visit should be clipped
1627  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1628  if len(clipIndex) == 1:
1629  keep = True
1630  keepIndex = [clipIndex[0]]
1631 
1632  # Test if there are clipped objects that overlap two different visits
1633  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1634  if len(clipIndex) == 2 and len(overlap) > 3:
1635  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1636  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1637  keep = True
1638  keepIndex = clipIndex
1639 
1640  if not keep:
1641  continue
1642 
1643  for index in keepIndex:
1644  globalIndex = indexList[index]
1645  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1646 
1647  clipIndices.append(numpy.array(indexList)[keepIndex])
1648  clipFootprints.append(footprint)
1649 
1650  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1651  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1652 
1653  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1654  maskClipValue, maskDetValue, coaddBBox):
1655  """Return individual warp footprints for large artifacts and append
1656  them to ``clipList`` in place.
1657 
1658  Identify big footprints composed of many sources in the coadd
1659  difference that may have originated in a large diffuse source in the
1660  coadd. We do this by indentifying all clipped footprints that overlap
1661  significantly with each source in all the coaddTempExps.
1662 
1663  Parameters
1664  ----------
1665  clipList : `list`
1666  List of alt mask SpanSets with clipping information. Modified.
1667  clipFootprints : `list`
1668  List of clipped footprints.
1669  clipIndices : `list`
1670  List of which entries in tempExpClipList each footprint belongs to.
1671  maskClipValue
1672  Mask value of clipped pixels.
1673  maskDetValue
1674  Mask value of detected pixels.
1675  coaddBBox : `lsst.geom.Box`
1676  BBox of the coadd and warps.
1677 
1678  Returns
1679  -------
1680  bigFootprintsCoadd : `list`
1681  List of big footprints
1682  """
1683  bigFootprintsCoadd = []
1684  ignoreMask = self.getBadPixelMask()
1685  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1686  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1687  for footprint in visitFootprints.getFootprints():
1688  footprint.spans.setMask(maskVisitDet, maskDetValue)
1689 
1690  # build a mask of clipped footprints that are in this visit
1691  clippedFootprintsVisit = []
1692  for foot, clipIndex in zip(clipFootprints, clipIndices):
1693  if index not in clipIndex:
1694  continue
1695  clippedFootprintsVisit.append(foot)
1696  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1697  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1698 
1699  bigFootprintsVisit = []
1700  for foot in visitFootprints.getFootprints():
1701  if foot.getArea() < self.config.minBigOverlap:
1702  continue
1703  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1704  if nCount > self.config.minBigOverlap:
1705  bigFootprintsVisit.append(foot)
1706  bigFootprintsCoadd.append(foot)
1707 
1708  for footprint in bigFootprintsVisit:
1709  clippedSpans["CLIPPED"].append(footprint.spans)
1710 
1711  return bigFootprintsCoadd
1712 
1713 
1715  psfMatchedWarps = pipeBase.connectionTypes.Input(
1716  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1717  "Only PSF-Matched Warps make sense for image subtraction. "
1718  "Therefore, they must be an additional declared input."),
1719  name="{inputCoaddName}Coadd_psfMatchedWarp",
1720  storageClass="ExposureF",
1721  dimensions=("tract", "patch", "skymap", "visit"),
1722  deferLoad=True,
1723  multiple=True
1724  )
1725  templateCoadd = pipeBase.connectionTypes.Output(
1726  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1727  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1728  name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1729  storageClass="ExposureF",
1730  dimensions=("tract", "patch", "skymap", "band"),
1731  )
1732 
1733  def __init__(self, *, config=None):
1734  super().__init__(config=config)
1735  if not config.assembleStaticSkyModel.doWrite:
1736  self.outputs.remove("templateCoadd")
1737  config.validate()
1738 
1739 
1740 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1741  pipelineConnections=CompareWarpAssembleCoaddConnections):
1742  assembleStaticSkyModel = pexConfig.ConfigurableField(
1743  target=AssembleCoaddTask,
1744  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1745  " naive/first-iteration model of the static sky.",
1746  )
1747  detect = pexConfig.ConfigurableField(
1748  target=SourceDetectionTask,
1749  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1750  )
1751  detectTemplate = pexConfig.ConfigurableField(
1752  target=SourceDetectionTask,
1753  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1754  )
1755  maskStreaks = pexConfig.ConfigurableField(
1756  target=MaskStreaksTask,
1757  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1758  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1759  "streakMaskName"
1760  )
1761  streakMaskName = pexConfig.Field(
1762  dtype=str,
1763  default="STREAK",
1764  doc="Name of mask bit used for streaks"
1765  )
1766  maxNumEpochs = pexConfig.Field(
1767  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1768  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1769  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1770  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1771  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1772  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1773  "than transient and not masked.",
1774  dtype=int,
1775  default=2
1776  )
1777  maxFractionEpochsLow = pexConfig.RangeField(
1778  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1779  "Effective maxNumEpochs = "
1780  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1781  dtype=float,
1782  default=0.4,
1783  min=0., max=1.,
1784  )
1785  maxFractionEpochsHigh = pexConfig.RangeField(
1786  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1787  "Effective maxNumEpochs = "
1788  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1789  dtype=float,
1790  default=0.03,
1791  min=0., max=1.,
1792  )
1793  spatialThreshold = pexConfig.RangeField(
1794  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1795  "temporal criteria. If 0, clip all. If 1, clip none.",
1796  dtype=float,
1797  default=0.5,
1798  min=0., max=1.,
1799  inclusiveMin=True, inclusiveMax=True
1800  )
1801  doScaleWarpVariance = pexConfig.Field(
1802  doc="Rescale Warp variance plane using empirical noise?",
1803  dtype=bool,
1804  default=True,
1805  )
1806  scaleWarpVariance = pexConfig.ConfigurableField(
1807  target=ScaleVarianceTask,
1808  doc="Rescale variance on warps",
1809  )
1810  doPreserveContainedBySource = pexConfig.Field(
1811  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1812  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1813  dtype=bool,
1814  default=True,
1815  )
1816  doPrefilterArtifacts = pexConfig.Field(
1817  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1818  "because they will be excluded anyway. This prevents them from contributing "
1819  "to the outlier epoch count image and potentially being labeled as persistant."
1820  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1821  dtype=bool,
1822  default=True
1823  )
1824  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1825  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1826  dtype=str,
1827  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1828  )
1829  prefilterArtifactsRatio = pexConfig.Field(
1830  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1831  dtype=float,
1832  default=0.05
1833  )
1834  doFilterMorphological = pexConfig.Field(
1835  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1836  "be streaks.",
1837  dtype=bool,
1838  default=False
1839  )
1840 
1841  def setDefaults(self):
1842  AssembleCoaddConfig.setDefaults(self)
1843  self.statistic = 'MEAN'
1845 
1846  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1847  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1848  if "EDGE" in self.badMaskPlanes:
1849  self.badMaskPlanes.remove('EDGE')
1850  self.removeMaskPlanes.append('EDGE')
1851  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1852  self.assembleStaticSkyModel.warpType = 'psfMatched'
1853  self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1854  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1855  self.assembleStaticSkyModel.sigmaClip = 2.5
1856  self.assembleStaticSkyModel.clipIter = 3
1857  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1858  self.assembleStaticSkyModel.doWrite = False
1859  self.detect.doTempLocalBackground = False
1860  self.detect.reEstimateBackground = False
1861  self.detect.returnOriginalFootprints = False
1862  self.detect.thresholdPolarity = "both"
1863  self.detect.thresholdValue = 5
1864  self.detect.minPixels = 4
1865  self.detect.isotropicGrow = True
1866  self.detect.thresholdType = "pixel_stdev"
1867  self.detect.nSigmaToGrow = 0.4
1868  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1869  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1870  self.detectTemplate.nSigmaToGrow = 2.4
1871  self.detectTemplate.doTempLocalBackground = False
1872  self.detectTemplate.reEstimateBackground = False
1873  self.detectTemplate.returnOriginalFootprints = False
1874 
1875  def validate(self):
1876  super().validate()
1877  if self.assembleStaticSkyModel.doNImage:
1878  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1879  "Please set assembleStaticSkyModel.doNImage=False")
1880 
1881  if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1882  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1883  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1884  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1885  "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1886 
1887 
1888 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1889  """Assemble a compareWarp coadded image from a set of warps
1890  by masking artifacts detected by comparing PSF-matched warps.
1891 
1892  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1893  we clip outliers). The problem with doing this is that when computing the
1894  coadd PSF at a given location, individual visit PSFs from visits with
1895  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1896  In this task, we correct for this behavior by creating a new badMaskPlane
1897  'CLIPPED' which marks pixels in the individual warps suspected to contain
1898  an artifact. We populate this plane on the input warps by comparing
1899  PSF-matched warps with a PSF-matched median coadd which serves as a
1900  model of the static sky. Any group of pixels that deviates from the
1901  PSF-matched template coadd by more than config.detect.threshold sigma,
1902  is an artifact candidate. The candidates are then filtered to remove
1903  variable sources and sources that are difficult to subtract such as
1904  bright stars. This filter is configured using the config parameters
1905  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1906  the maximum fraction of epochs that the deviation can appear in and still
1907  be considered an artifact. The spatialThreshold is the maximum fraction of
1908  pixels in the footprint of the deviation that appear in other epochs
1909  (where other epochs is defined by the temporalThreshold). If the deviant
1910  region meets this criteria of having a significant percentage of pixels
1911  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1912  set in the mask. These regions will not contribute to the final coadd.
1913  Furthermore, any routine to determine the coadd PSF can now be cognizant
1914  of clipped regions. Note that the algorithm implemented by this task is
1915  preliminary and works correctly for HSC data. Parameter modifications and
1916  or considerable redesigning of the algorithm is likley required for other
1917  surveys.
1918 
1919  ``CompareWarpAssembleCoaddTask`` sub-classes
1920  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1921  as a subtask to generate the TemplateCoadd (the model of the static sky).
1922 
1923  Notes
1924  -----
1925  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1926  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1927  ``baseDebug`` for more about ``debug.py`` files.
1928 
1929  This task supports the following debug variables:
1930 
1931  - ``saveCountIm``
1932  If True then save the Epoch Count Image as a fits file in the `figPath`
1933  - ``figPath``
1934  Path to save the debug fits images and figures
1935 
1936  For example, put something like:
1937 
1938  .. code-block:: python
1939 
1940  import lsstDebug
1941  def DebugInfo(name):
1942  di = lsstDebug.getInfo(name)
1943  if name == "lsst.pipe.tasks.assembleCoadd":
1944  di.saveCountIm = True
1945  di.figPath = "/desired/path/to/debugging/output/images"
1946  return di
1947  lsstDebug.Info = DebugInfo
1948 
1949  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1950  ``--debug`` flag. Some subtasks may have their own debug variables;
1951  see individual Task documentation.
1952 
1953  Examples
1954  --------
1955  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1956  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1957  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1958  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1959  and filter to be coadded (specified using
1960  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1961  along with a list of coaddTempExps to attempt to coadd (specified using
1962  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1963  Only the warps that cover the specified tract and patch will be coadded.
1964  A list of the available optional arguments can be obtained by calling
1965  ``assembleCoadd.py`` with the ``--help`` command line argument:
1966 
1967  .. code-block:: none
1968 
1969  assembleCoadd.py --help
1970 
1971  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1972  context of multi-band processing, we will generate the HSC-I & -R band
1973  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1974  To begin, assuming that the lsst stack has been already set up, we must
1975  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1976  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1977  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1978  directory. To begin assembling the coadds, we must first
1979 
1980  - processCcd
1981  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1982  - makeSkyMap
1983  create a skymap that covers the area of the sky present in the raw exposures
1984  - makeCoaddTempExp
1985  warp the individual calibrated exposures to the tangent plane of the coadd
1986 
1987  We can perform all of these steps by running
1988 
1989  .. code-block:: none
1990 
1991  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1992 
1993  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1994  warped data, we call ``assembleCoadd.py`` as follows:
1995 
1996  .. code-block:: none
1997 
1998  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1999  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2000  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2001  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2002  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2003  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2004  --selectId visit=903988 ccd=24
2005 
2006  This will process the HSC-I band data. The results are written in
2007  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2008  """
2009  ConfigClass = CompareWarpAssembleCoaddConfig
2010  _DefaultName = "compareWarpAssembleCoadd"
2011 
2012  def __init__(self, *args, **kwargs):
2013  AssembleCoaddTask.__init__(self, *args, **kwargs)
2014  self.makeSubtask("assembleStaticSkyModel")
2015  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2016  self.makeSubtask("detect", schema=detectionSchema)
2017  if self.config.doPreserveContainedBySource:
2018  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2019  if self.config.doScaleWarpVariance:
2020  self.makeSubtask("scaleWarpVariance")
2021  if self.config.doFilterMorphological:
2022  self.makeSubtask("maskStreaks")
2023 
2024  @utils.inheritDoc(AssembleCoaddTask)
2025  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2026  """
2027  Generate a templateCoadd to use as a naive model of static sky to
2028  subtract from PSF-Matched warps.
2029 
2030  Returns
2031  -------
2032  result : `lsst.pipe.base.Struct`
2033  Result struct with components:
2034 
2035  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2036  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2037  """
2038  # Ensure that psfMatchedWarps are used as input warps for template generation
2039  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2040  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2041 
2042  # Because subtasks don't have connections we have to make one.
2043  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2044  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2045  if self.config.assembleStaticSkyModel.doWrite:
2046  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2047  # Remove template coadd from both subtask's and main tasks outputs,
2048  # because it is handled by the subtask as `coaddExposure`
2049  del outputRefs.templateCoadd
2050  del staticSkyModelOutputRefs.templateCoadd
2051 
2052  # A PSF-Matched nImage does not exist as a dataset type
2053  if 'nImage' in staticSkyModelOutputRefs.keys():
2054  del staticSkyModelOutputRefs.nImage
2055 
2056  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2057  staticSkyModelOutputRefs)
2058  if templateCoadd is None:
2059  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2060 
2061  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2062  nImage=templateCoadd.nImage,
2063  warpRefList=templateCoadd.warpRefList,
2064  imageScalerList=templateCoadd.imageScalerList,
2065  weightList=templateCoadd.weightList)
2066 
2067  @utils.inheritDoc(AssembleCoaddTask)
2068  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2069  """
2070  Generate a templateCoadd to use as a naive model of static sky to
2071  subtract from PSF-Matched warps.
2072 
2073  Returns
2074  -------
2075  result : `lsst.pipe.base.Struct`
2076  Result struct with components:
2077 
2078  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2079  - ``nImage``: N Image (``lsst.afw.image.Image``)
2080  """
2081  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2082  if templateCoadd is None:
2083  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2084 
2085  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2086  nImage=templateCoadd.nImage,
2087  warpRefList=templateCoadd.warpRefList,
2088  imageScalerList=templateCoadd.imageScalerList,
2089  weightList=templateCoadd.weightList)
2090 
2091  def _noTemplateMessage(self, warpType):
2092  warpName = (warpType[0].upper() + warpType[1:])
2093  message = """No %(warpName)s warps were found to build the template coadd which is
2094  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2095  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2096  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2097 
2098  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2099  another algorithm like:
2100 
2101  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2102  config.assemble.retarget(SafeClipAssembleCoaddTask)
2103  """ % {"warpName": warpName}
2104  return message
2105 
2106  @utils.inheritDoc(AssembleCoaddTask)
2107  @pipeBase.timeMethod
2108  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2109  supplementaryData, *args, **kwargs):
2110  """Assemble the coadd.
2111 
2112  Find artifacts and apply them to the warps' masks creating a list of
2113  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2114  plane. Then pass these alternative masks to the base class's `run`
2115  method.
2116 
2117  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2118  that must contain a ``templateCoadd`` that serves as the
2119  model of the static sky.
2120  """
2121 
2122  # Check and match the order of the supplementaryData
2123  # (PSF-matched) inputs to the order of the direct inputs,
2124  # so that the artifact mask is applied to the right warp
2125  dataIds = [ref.dataId for ref in tempExpRefList]
2126  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2127 
2128  if dataIds != psfMatchedDataIds:
2129  self.log.info("Reordering and or/padding PSF-matched visit input list")
2130  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2131  psfMatchedDataIds, dataIds)
2132  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2133  psfMatchedDataIds, dataIds)
2134 
2135  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2136  spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2137  supplementaryData.warpRefList,
2138  supplementaryData.imageScalerList)
2139 
2140  badMaskPlanes = self.config.badMaskPlanes[:]
2141  badMaskPlanes.append("CLIPPED")
2142  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2143 
2144  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2145  spanSetMaskList, mask=badPixelMask)
2146 
2147  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2148  # Psf-Matching moves the real edge inwards
2149  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2150  return result
2151 
2152  def applyAltEdgeMask(self, mask, altMaskList):
2153  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2154 
2155  Parameters
2156  ----------
2157  mask : `lsst.afw.image.Mask`
2158  Original mask.
2159  altMaskList : `list`
2160  List of Dicts containing ``spanSet`` lists.
2161  Each element contains the new mask plane name (e.g. "CLIPPED
2162  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2163  the mask.
2164  """
2165  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2166  for visitMask in altMaskList:
2167  if "EDGE" in visitMask:
2168  for spanSet in visitMask['EDGE']:
2169  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2170 
2171  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2172  """Find artifacts.
2173 
2174  Loop through warps twice. The first loop builds a map with the count
2175  of how many epochs each pixel deviates from the templateCoadd by more
2176  than ``config.chiThreshold`` sigma. The second loop takes each
2177  difference image and filters the artifacts detected in each using
2178  count map to filter out variable sources and sources that are
2179  difficult to subtract cleanly.
2180 
2181  Parameters
2182  ----------
2183  templateCoadd : `lsst.afw.image.Exposure`
2184  Exposure to serve as model of static sky.
2185  tempExpRefList : `list`
2186  List of data references to warps.
2187  imageScalerList : `list`
2188  List of image scalers.
2189 
2190  Returns
2191  -------
2192  altMasks : `list`
2193  List of dicts containing information about CLIPPED
2194  (i.e., artifacts), NO_DATA, and EDGE pixels.
2195  """
2196 
2197  self.log.debug("Generating Count Image, and mask lists.")
2198  coaddBBox = templateCoadd.getBBox()
2199  slateIm = afwImage.ImageU(coaddBBox)
2200  epochCountImage = afwImage.ImageU(coaddBBox)
2201  nImage = afwImage.ImageU(coaddBBox)
2202  spanSetArtifactList = []
2203  spanSetNoDataMaskList = []
2204  spanSetEdgeList = []
2205  spanSetBadMorphoList = []
2206  badPixelMask = self.getBadPixelMask()
2207 
2208  # mask of the warp diffs should = that of only the warp
2209  templateCoadd.mask.clearAllMaskPlanes()
2210 
2211  if self.config.doPreserveContainedBySource:
2212  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2213  else:
2214  templateFootprints = None
2215 
2216  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2217  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2218  if warpDiffExp is not None:
2219  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2220  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2221  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2222  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2223  fpSet.positive.merge(fpSet.negative)
2224  footprints = fpSet.positive
2225  slateIm.set(0)
2226  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2227 
2228  # Remove artifacts due to defects before they contribute to the epochCountImage
2229  if self.config.doPrefilterArtifacts:
2230  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2231 
2232  # Clear mask before adding prefiltered spanSets
2233  self.detect.clearMask(warpDiffExp.mask)
2234  for spans in spanSetList:
2235  spans.setImage(slateIm, 1, doClip=True)
2236  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2237  epochCountImage += slateIm
2238 
2239  if self.config.doFilterMorphological:
2240  maskName = self.config.streakMaskName
2241  _ = self.maskStreaks.run(warpDiffExp)
2242  streakMask = warpDiffExp.mask
2243  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2244  streakMask.getPlaneBitMask(maskName)).split()
2245 
2246  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2247  # undergo a second convolution. Pixels with data in the direct warp
2248  # but not in the PSF-matched warp will not have their artifacts detected.
2249  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2250  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2251  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2252  nansMask.setXY0(warpDiffExp.getXY0())
2253  edgeMask = warpDiffExp.mask
2254  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2255  edgeMask.getPlaneBitMask("EDGE")).split()
2256  else:
2257  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2258  # In this case, mask the whole epoch
2259  nansMask = afwImage.MaskX(coaddBBox, 1)
2260  spanSetList = []
2261  spanSetEdgeMask = []
2262  spanSetStreak = []
2263 
2264  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2265 
2266  spanSetNoDataMaskList.append(spanSetNoDataMask)
2267  spanSetArtifactList.append(spanSetList)
2268  spanSetEdgeList.append(spanSetEdgeMask)
2269  if self.config.doFilterMorphological:
2270  spanSetBadMorphoList.append(spanSetStreak)
2271 
2272  if lsstDebug.Info(__name__).saveCountIm:
2273  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2274  epochCountImage.writeFits(path)
2275 
2276  for i, spanSetList in enumerate(spanSetArtifactList):
2277  if spanSetList:
2278  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2279  templateFootprints)
2280  spanSetArtifactList[i] = filteredSpanSetList
2281  if self.config.doFilterMorphological:
2282  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2283 
2284  altMasks = []
2285  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2286  altMasks.append({'CLIPPED': artifacts,
2287  'NO_DATA': noData,
2288  'EDGE': edge})
2289  return altMasks
2290 
2291  def prefilterArtifacts(self, spanSetList, exp):
2292  """Remove artifact candidates covered by bad mask plane.
2293 
2294  Any future editing of the candidate list that does not depend on
2295  temporal information should go in this method.
2296 
2297  Parameters
2298  ----------
2299  spanSetList : `list`
2300  List of SpanSets representing artifact candidates.
2301  exp : `lsst.afw.image.Exposure`
2302  Exposure containing mask planes used to prefilter.
2303 
2304  Returns
2305  -------
2306  returnSpanSetList : `list`
2307  List of SpanSets with artifacts.
2308  """
2309  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2310  goodArr = (exp.mask.array & badPixelMask) == 0
2311  returnSpanSetList = []
2312  bbox = exp.getBBox()
2313  x0, y0 = exp.getXY0()
2314  for i, span in enumerate(spanSetList):
2315  y, x = span.clippedTo(bbox).indices()
2316  yIndexLocal = numpy.array(y) - y0
2317  xIndexLocal = numpy.array(x) - x0
2318  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2319  if goodRatio > self.config.prefilterArtifactsRatio:
2320  returnSpanSetList.append(span)
2321  return returnSpanSetList
2322 
2323  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2324  """Filter artifact candidates.
2325 
2326  Parameters
2327  ----------
2328  spanSetList : `list`
2329  List of SpanSets representing artifact candidates.
2330  epochCountImage : `lsst.afw.image.Image`
2331  Image of accumulated number of warpDiff detections.
2332  nImage : `lsst.afw.image.Image`
2333  Image of the accumulated number of total epochs contributing.
2334 
2335  Returns
2336  -------
2337  maskSpanSetList : `list`
2338  List of SpanSets with artifacts.
2339  """
2340 
2341  maskSpanSetList = []
2342  x0, y0 = epochCountImage.getXY0()
2343  for i, span in enumerate(spanSetList):
2344  y, x = span.indices()
2345  yIdxLocal = [y1 - y0 for y1 in y]
2346  xIdxLocal = [x1 - x0 for x1 in x]
2347  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2348  totalN = nImage.array[yIdxLocal, xIdxLocal]
2349 
2350  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2351  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2352  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2353  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2354  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2355  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2356  & (outlierN <= effectiveMaxNumEpochs))
2357  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2358  if percentBelowThreshold > self.config.spatialThreshold:
2359  maskSpanSetList.append(span)
2360 
2361  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2362  # If a candidate is contained by a footprint on the template coadd, do not clip
2363  filteredMaskSpanSetList = []
2364  for span in maskSpanSetList:
2365  doKeep = True
2366  for footprint in footprintsToExclude.positive.getFootprints():
2367  if footprint.spans.contains(span):
2368  doKeep = False
2369  break
2370  if doKeep:
2371  filteredMaskSpanSetList.append(span)
2372  maskSpanSetList = filteredMaskSpanSetList
2373 
2374  return maskSpanSetList
2375 
2376  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2377  """Fetch a warp from the butler and return a warpDiff.
2378 
2379  Parameters
2380  ----------
2381  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2382  Butler dataRef for the warp.
2383  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2384  An image scaler object.
2385  templateCoadd : `lsst.afw.image.Exposure`
2386  Exposure to be substracted from the scaled warp.
2387 
2388  Returns
2389  -------
2390  warp : `lsst.afw.image.Exposure`
2391  Exposure of the image difference between the warp and template.
2392  """
2393 
2394  # If the PSF-Matched warp did not exist for this direct warp
2395  # None is holding its place to maintain order in Gen 3
2396  if warpRef is None:
2397  return None
2398  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2399  warpName = self.getTempExpDatasetName('psfMatched')
2400  if not isinstance(warpRef, DeferredDatasetHandle):
2401  if not warpRef.datasetExists(warpName):
2402  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2403  return None
2404  warp = warpRef.get(datasetType=warpName, immediate=True)
2405  # direct image scaler OK for PSF-matched Warp
2406  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2407  mi = warp.getMaskedImage()
2408  if self.config.doScaleWarpVariance:
2409  try:
2410  self.scaleWarpVariance.run(mi)
2411  except Exception as exc:
2412  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2413  mi -= templateCoadd.getMaskedImage()
2414  return warp
2415 
2416  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2417  """Return a path to which to write debugging output.
2418 
2419  Creates a hyphen-delimited string of dataId values for simple filenames.
2420 
2421  Parameters
2422  ----------
2423  prefix : `str`
2424  Prefix for filename.
2425  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2426  Butler dataRef to make the path from.
2427  coaddLevel : `bool`, optional.
2428  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2429  'filter', but no 'visit').
2430 
2431  Returns
2432  -------
2433  result : `str`
2434  Path for debugging output.
2435  """
2436  if coaddLevel:
2437  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2438  else:
2439  keys = warpRef.dataId.keys()
2440  keyList = sorted(keys, reverse=True)
2441  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2442  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2443  return os.path.join(directory, filename)
2444 
2445 
2446 def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2447  """Match the order of one list to another, padding if necessary
2448 
2449  Parameters
2450  ----------
2451  inputList : list
2452  List to be reordered and padded. Elements can be any type.
2453  inputKeys : iterable
2454  Iterable of values to be compared with outputKeys.
2455  Length must match `inputList`
2456  outputKeys : iterable
2457  Iterable of values to be compared with inputKeys.
2458  padWith :
2459  Any value to be inserted where inputKey not in outputKeys
2460 
2461  Returns
2462  -------
2463  list
2464  Copy of inputList reordered per outputKeys and padded with `padWith`
2465  so that the length matches length of outputKeys.
2466  """
2467  outputList = []
2468  for d in outputKeys:
2469  if d in inputKeys:
2470  outputList.append(inputList[inputKeys.index(d)])
2471  else:
2472  outputList.append(padWith)
2473  return outputList
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.filterArtifacts
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
Definition: assembleCoadd.py:2323
lsst::afw::image
lsst.pipe.tasks.assembleCoadd.getTempExpRefList
def getTempExpRefList(self, patchRef, calExpRefList)
Definition: assembleCoadd.py:589
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig
Definition: assembleCoadd.py:1741
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig
Definition: assembleCoadd.py:1232
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.makeSupplementaryData
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
Definition: assembleCoadd.py:2068
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Definition: assembleCoadd.py:1435
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
Definition: assembleCoadd.py:2108
lsst.pipe.tasks.assembleCoadd.assembleSubregion
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Definition: assembleCoadd.py:863
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.validate
def validate(self)
Definition: assembleCoadd.py:1875
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.__init__
def __init__(self, *args, **kwargs)
Definition: assembleCoadd.py:1426
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask
Definition: assembleCoadd.py:1314
lsst.pipe.tasks.assembleCoadd.processResults
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
Definition: assembleCoadd.py:526
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.clipDetection
clipDetection
Definition: assembleCoadd.py:1243
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.findArtifacts
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Definition: assembleCoadd.py:2171
lsst.pipe.tasks.coaddBase.makeSkyInfo
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.statistic
statistic
Definition: assembleCoadd.py:1843
lsst.pipe.tasks.assembleCoadd.AssembleCoaddConnections
Definition: assembleCoadd.py:57
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._noTemplateMessage
def _noTemplateMessage(self, warpType)
Definition: assembleCoadd.py:2091
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.detectTemplate
detectTemplate
Definition: assembleCoadd.py:1751
lsst.pipe.tasks.assembleCoadd.reorderAndPadList
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: assembleCoadd.py:2446
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.warpType
warpType
Definition: assembleCoadd.py:1881
lsst.pipe.tasks.assembleCoadd.removeMaskPlanes
def removeMaskPlanes(self, maskedImage)
Definition: assembleCoadd.py:940
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.detectClip
def detectClip(self, exp, tempExpRefList)
Definition: assembleCoadd.py:1527
lsst.pipe.tasks.assembleCoadd.assembleMetadata
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Definition: assembleCoadd.py:802
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.sigmaClip
sigmaClip
Definition: assembleCoadd.py:1294
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.assembleMeanClipCoadd
assembleMeanClipCoadd
Definition: assembleCoadd.py:1239
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask
Definition: assembleCoadd.py:1888
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConnections.__init__
def __init__(self, *config=None)
Definition: assembleCoadd.py:1733
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.prefilterArtifacts
def prefilterArtifacts(self, spanSetList, exp)
Definition: assembleCoadd.py:2291
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._readAndComputeWarpDiff
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Definition: assembleCoadd.py:2376
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:721
lsst.pipe.tasks.assembleCoadd.prepareStats
def prepareStats(self, mask=None)
Definition: assembleCoadd.py:688
lsstDebug::Info
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.detect
detect
Definition: assembleCoadd.py:1747
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.doUsePsfMatchedPolygons
doUsePsfMatchedPolygons
Definition: assembleCoadd.py:1844
lsst.pipe.tasks.assembleCoadd.makeSupplementaryData
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
Definition: assembleCoadd.py:546
lsst::afw::table
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConnections
Definition: assembleCoadd.py:1714
lsst::utils
lsst.pipe.tasks.assembleCoadd.setInexactPsf
def setInexactPsf(self, mask)
Definition: assembleCoadd.py:1103
lsst::pex::config
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.assembleMeanCoadd
assembleMeanCoadd
Definition: assembleCoadd.py:1235
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.buildDifferenceImage
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Definition: assembleCoadd.py:1491
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.doSigmaClip
doSigmaClip
Definition: assembleCoadd.py:1306
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.detectClipBig
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Definition: assembleCoadd.py:1653
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.applyAltEdgeMask
def applyAltEdgeMask(self, mask, altMaskList)
Definition: assembleCoadd.py:2152
lsst::afw::detection
lsst.pipe.tasks.coaddHelpers.getGroupDataRef
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
lsst::geom
lsst.pipe.tasks.assembleCoadd.countMaskFromFootprint
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Definition: assembleCoadd.py:1199
lsst.pipe.tasks.assembleCoadd.readBrightObjectMasks
def readBrightObjectMasks(self, dataRef)
Definition: assembleCoadd.py:1039
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.setDefaults
def setDefaults(self)
Definition: assembleCoadd.py:1276
lsst.pipe.tasks.assembleCoadd.AssembleCoaddDataIdContainer
Definition: assembleCoadd.py:1171
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.validate
def validate(self)
Definition: assembleCoadd.py:1302
lsst.pipe.tasks.assembleCoadd.prepareInputs
def prepareInputs(self, refList)
Definition: assembleCoadd.py:613
lsst::afw::math
lsst.pipe.tasks.coaddBase.makeCoaddSuffix
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:336
lsst.pipe.tasks.assembleCoadd.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:565
Point< int, 2 >
lsst.pipe.tasks.coaddBase.CoaddBaseTask
Base class for coaddition.
Definition: coaddBase.py:131
lsst.pipe.tasks.assembleCoadd.setBrightObjectMasks
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
Definition: assembleCoadd.py:1061
lsst::geom::Box2I
lsst::skymap
lsst::pex::exceptions
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._dataRef2DebugPath
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Definition: assembleCoadd.py:2416
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.clipIter
clipIter
Definition: assembleCoadd.py:1295
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.__init__
def __init__(self, *args, **kwargs)
Definition: assembleCoadd.py:2012
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.statistic
statistic
Definition: assembleCoadd.py:1296
lsst::geom::Box2D
lsst.pipe::base
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.assembleStaticSkyModel
assembleStaticSkyModel
Definition: assembleCoadd.py:1742
lsst::log
lsst.pipe.tasks.assembleCoadd.applyAltMaskPlanes
def applyAltMaskPlanes(self, mask, altMaskSpans)
Definition: assembleCoadd.py:985
lsst::meas::algorithms
lsst.pipe.tasks.coaddHelpers.groupPatchExposures
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:59
Extent< int, 2 >
lsst::coadd::utils
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.setDefaults
def setDefaults(self)
Definition: assembleCoadd.py:1841
lsst.pipe.tasks.assembleCoadd.AssembleCoaddDataIdContainer.makeDataRefList
def makeDataRefList(self, namespace)
Definition: assembleCoadd.py:1175
lsst.pipe.tasks.assembleCoadd.shrinkValidPolygons
def shrinkValidPolygons(self, coaddInputs)
Definition: assembleCoadd.py:1017
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:2025
lsst::afw::geom