lsst.pipe.tasks  20.0.0-21-g7fb4cbd3+53a1050ad1
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41 from .interpImage import InterpImageTask
42 from .scaleZeroPoint import ScaleZeroPointTask
43 from .coaddHelpers import groupPatchExposures, getGroupDataRef
44 from .scaleVariance import ScaleVarianceTask
45 from .maskStreaks import MaskStreaksTask
46 from lsst.meas.algorithms import SourceDetectionTask
47 from lsst.daf.butler import DeferredDatasetHandle
48 
49 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
50  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
51  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
52 
53 
54 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
55  dimensions=("tract", "patch", "abstract_filter", "skymap"),
56  defaultTemplates={"inputCoaddName": "deep",
57  "outputCoaddName": "deep",
58  "warpType": "direct",
59  "warpTypeSuffix": "",
60  "fakesType": ""}):
61  inputWarps = pipeBase.connectionTypes.Input(
62  doc=("Input list of warps to be assemebled i.e. stacked."
63  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
64  name="{inputCoaddName}Coadd_{warpType}Warp",
65  storageClass="ExposureF",
66  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
67  deferLoad=True,
68  multiple=True
69  )
70  skyMap = pipeBase.connectionTypes.Input(
71  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
72  name="{inputCoaddName}Coadd_skyMap",
73  storageClass="SkyMap",
74  dimensions=("skymap", ),
75  )
76  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
77  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
78  " BRIGHT_OBJECT."),
79  name="brightObjectMask",
80  storageClass="ObjectMaskCatalog",
81  dimensions=("tract", "patch", "skymap", "abstract_filter"),
82  )
83  coaddExposure = pipeBase.connectionTypes.Output(
84  doc="Output coadded exposure, produced by stacking input warps",
85  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
86  storageClass="ExposureF",
87  dimensions=("tract", "patch", "skymap", "abstract_filter"),
88  )
89  nImage = pipeBase.connectionTypes.Output(
90  doc="Output image of number of input images per pixel",
91  name="{outputCoaddName}Coadd_nImage",
92  storageClass="ImageU",
93  dimensions=("tract", "patch", "skymap", "abstract_filter"),
94  )
95 
96  def __init__(self, *, config=None):
97  # Override the connection's name template with config to replicate Gen2 behavior
98  config.connections.warpType = config.warpType
99  config.connections.warpTypeSuffix = makeCoaddSuffix(config.warpType)
100 
101  if config.hasFakes:
102  config.connections.fakesType = "_fakes"
103 
104  super().__init__(config=config)
105 
106  if not config.doMaskBrightObjects:
107  self.prerequisiteInputs.remove("brightObjectMask")
108 
109  if not config.doNImage:
110  self.outputs.remove("nImage")
111 
112 
113 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
114  pipelineConnections=AssembleCoaddConnections):
115  """Configuration parameters for the `AssembleCoaddTask`.
116 
117  Notes
118  -----
119  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
120  only set the bitplane config.brightObjectMaskName. To make this useful you
121  *must* also configure the flags.pixel algorithm, for example by adding
122 
123  .. code-block:: none
124 
125  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
126  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
127 
128  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
129  """
130  warpType = pexConfig.Field(
131  doc="Warp name: one of 'direct' or 'psfMatched'",
132  dtype=str,
133  default="direct",
134  )
135  subregionSize = pexConfig.ListField(
136  dtype=int,
137  doc="Width, height of stack subregion size; "
138  "make small enough that a full stack of images will fit into memory at once.",
139  length=2,
140  default=(2000, 2000),
141  )
142  statistic = pexConfig.Field(
143  dtype=str,
144  doc="Main stacking statistic for aggregating over the epochs.",
145  default="MEANCLIP",
146  )
147  doSigmaClip = pexConfig.Field(
148  dtype=bool,
149  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
150  default=False,
151  )
152  sigmaClip = pexConfig.Field(
153  dtype=float,
154  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
155  default=3.0,
156  )
157  clipIter = pexConfig.Field(
158  dtype=int,
159  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
160  default=2,
161  )
162  calcErrorFromInputVariance = pexConfig.Field(
163  dtype=bool,
164  doc="Calculate coadd variance from input variance by stacking statistic."
165  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
166  default=True,
167  )
168  scaleZeroPoint = pexConfig.ConfigurableField(
169  target=ScaleZeroPointTask,
170  doc="Task to adjust the photometric zero point of the coadd temp exposures",
171  )
172  doInterp = pexConfig.Field(
173  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
174  dtype=bool,
175  default=True,
176  )
177  interpImage = pexConfig.ConfigurableField(
178  target=InterpImageTask,
179  doc="Task to interpolate (and extrapolate) over NaN pixels",
180  )
181  doWrite = pexConfig.Field(
182  doc="Persist coadd?",
183  dtype=bool,
184  default=True,
185  )
186  doNImage = pexConfig.Field(
187  doc="Create image of number of contributing exposures for each pixel",
188  dtype=bool,
189  default=False,
190  )
191  doUsePsfMatchedPolygons = pexConfig.Field(
192  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
193  dtype=bool,
194  default=False,
195  )
196  maskPropagationThresholds = pexConfig.DictField(
197  keytype=str,
198  itemtype=float,
199  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
200  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
201  "would have contributed exceeds this value."),
202  default={"SAT": 0.1},
203  )
204  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
205  doc="Mask planes to remove before coadding")
206  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
207  doc="Set mask and flag bits for bright objects?")
208  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
209  doc="Name of mask bit used for bright objects")
210  coaddPsf = pexConfig.ConfigField(
211  doc="Configuration for CoaddPsf",
212  dtype=measAlg.CoaddPsfConfig,
213  )
214  doAttachTransmissionCurve = pexConfig.Field(
215  dtype=bool, default=False, optional=False,
216  doc=("Attach a piecewise TransmissionCurve for the coadd? "
217  "(requires all input Exposures to have TransmissionCurves).")
218  )
219  hasFakes = pexConfig.Field(
220  dtype=bool,
221  default=False,
222  doc="Should be set to True if fake sources have been inserted into the input data."
223  )
224 
225  def setDefaults(self):
226  super().setDefaults()
227  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
228 
229  def validate(self):
230  super().validate()
231  if self.doPsfMatch:
232  # Backwards compatibility.
233  # Configs do not have loggers
234  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
235  self.warpType = 'psfMatched'
236  if self.doSigmaClip and self.statistic != "MEANCLIP":
237  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
238  self.statistic = "MEANCLIP"
239  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
240  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
241  "compute and set a non-zero coadd variance estimate." % (self.statistic))
242 
243  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
244  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
245  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
246  if str(k) not in unstackableStats]
247  raise ValueError("statistic %s is not allowed. Please choose one of %s."
248  % (self.statistic, stackableStats))
249 
250 
251 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
252  """Assemble a coadded image from a set of warps (coadded temporary exposures).
253 
254  We want to assemble a coadded image from a set of Warps (also called
255  coadded temporary exposures or ``coaddTempExps``).
256  Each input Warp covers a patch on the sky and corresponds to a single
257  run/visit/exposure of the covered patch. We provide the task with a list
258  of Warps (``selectDataList``) from which it selects Warps that cover the
259  specified patch (pointed at by ``dataRef``).
260  Each Warp that goes into a coadd will typically have an independent
261  photometric zero-point. Therefore, we must scale each Warp to set it to
262  a common photometric zeropoint. WarpType may be one of 'direct' or
263  'psfMatched', and the boolean configs `config.makeDirect` and
264  `config.makePsfMatched` set which of the warp types will be coadded.
265  The coadd is computed as a mean with optional outlier rejection.
266  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
267  Finally, Warps can have bad 'NaN' pixels which received no input from the
268  source calExps. We interpolate over these bad (NaN) pixels.
269 
270  `AssembleCoaddTask` uses several sub-tasks. These are
271 
272  - `ScaleZeroPointTask`
273  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
274  - `InterpImageTask`
275  - interpolate across bad pixels (NaN) in the final coadd
276 
277  You can retarget these subtasks if you wish.
278 
279  Notes
280  -----
281  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
282  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
283  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
284  no debug variables of its own. Some of the subtasks may support debug
285  variables. See the documentation for the subtasks for further information.
286 
287  Examples
288  --------
289  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
290  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
291  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
292  inputs: a data reference to the tract patch and filter to be coadded, and
293  a list of Warps to attempt to coadd. These are specified using ``--id`` and
294  ``--selectId``, respectively:
295 
296  .. code-block:: none
297 
298  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
299  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
300 
301  Only the Warps that cover the specified tract and patch will be coadded.
302  A list of the available optional arguments can be obtained by calling
303  ``assembleCoadd.py`` with the ``--help`` command line argument:
304 
305  .. code-block:: none
306 
307  assembleCoadd.py --help
308 
309  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
310  multi-band processing, we will generate the HSC-I & -R band coadds from
311  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
312  assuming that the lsst stack has been already set up, we must set up the
313  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
314  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
315  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
316  coadds, we must first
317 
318  - processCcd
319  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
320  - makeSkyMap
321  - create a skymap that covers the area of the sky present in the raw exposures
322  - makeCoaddTempExp
323  - warp the individual calibrated exposures to the tangent plane of the coadd
324 
325  We can perform all of these steps by running
326 
327  .. code-block:: none
328 
329  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
330 
331  This will produce warped exposures for each visit. To coadd the warped
332  data, we call assembleCoadd.py as follows:
333 
334  .. code-block:: none
335 
336  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
337  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
338  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
339  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
340  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
341  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
342  --selectId visit=903988 ccd=24
343 
344  that will process the HSC-I band data. The results are written in
345  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
346 
347  You may also choose to run:
348 
349  .. code-block:: none
350 
351  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
352  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
353  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
354  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
355  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
356  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
357  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
358  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
359 
360  to generate the coadd for the HSC-R band if you are interested in
361  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
362  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
363  rather than `AssembleCoaddTask` to make the coadd.
364  """
365  ConfigClass = AssembleCoaddConfig
366  _DefaultName = "assembleCoadd"
367 
368  def __init__(self, *args, **kwargs):
369  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
370  if args:
371  argNames = ["config", "name", "parentTask", "log"]
372  kwargs.update({k: v for k, v in zip(argNames, args)})
373  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
374  "PipelineTask will not take positional args" % argNames, FutureWarning)
375 
376  super().__init__(**kwargs)
377  self.makeSubtask("interpImage")
378  self.makeSubtask("scaleZeroPoint")
379 
380  if self.config.doMaskBrightObjects:
381  mask = afwImage.Mask()
382  try:
383  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
384  except pexExceptions.LsstCppException:
385  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
386  mask.getMaskPlaneDict().keys())
387  del mask
388 
389  self.warpType = self.config.warpType
390 
391  @utils.inheritDoc(pipeBase.PipelineTask)
392  def runQuantum(self, butlerQC, inputRefs, outputRefs):
393  # Docstring to be formatted with info from PipelineTask.runQuantum
394  """
395  Notes
396  -----
397  Assemble a coadd from a set of Warps.
398 
399  PipelineTask (Gen3) entry point to Coadd a set of Warps.
400  Analogous to `runDataRef`, it prepares all the data products to be
401  passed to `run`, and processes the results before returning a struct
402  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
403  Therefore, its inputs are accessed subregion by subregion
404  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
405  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
406  correspond to an update in `runDataRef` while both entry points
407  are used.
408  """
409  inputData = butlerQC.get(inputRefs)
410 
411  # Construct skyInfo expected by run
412  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
413  skyMap = inputData["skyMap"]
414  outputDataId = butlerQC.quantum.dataId
415 
416  inputData['skyInfo'] = makeSkyInfo(skyMap,
417  tractId=outputDataId['tract'],
418  patchId=outputDataId['patch'])
419 
420  # Construct list of input Deferred Datasets
421  # These quack a bit like like Gen2 DataRefs
422  warpRefList = inputData['inputWarps']
423  # Perform same middle steps as `runDataRef` does
424  inputs = self.prepareInputs(warpRefList)
425  self.log.info("Found %d %s", len(inputs.tempExpRefList),
426  self.getTempExpDatasetName(self.warpType))
427  if len(inputs.tempExpRefList) == 0:
428  self.log.warn("No coadd temporary exposures found")
429  return
430 
431  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
432  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
433  inputs.weightList, supplementaryData=supplementaryData)
434 
435  inputData.setdefault('brightObjectMask', None)
436  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
437 
438  if self.config.doWrite:
439  butlerQC.put(retStruct, outputRefs)
440  return retStruct
441 
442  @pipeBase.timeMethod
443  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
444  """Assemble a coadd from a set of Warps.
445 
446  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
447  Compute weights to be applied to each Warp and
448  find scalings to match the photometric zeropoint to a reference Warp.
449  Assemble the Warps using `run`. Interpolate over NaNs and
450  optionally write the coadd to disk. Return the coadded exposure.
451 
452  Parameters
453  ----------
454  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
455  Data reference defining the patch for coaddition and the
456  reference Warp (if ``config.autoReference=False``).
457  Used to access the following data products:
458  - ``self.config.coaddName + "Coadd_skyMap"``
459  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
460  - ``self.config.coaddName + "Coadd"``
461  selectDataList : `list`
462  List of data references to Calexps. Data to be coadded will be
463  selected from this list based on overlap with the patch defined
464  by dataRef, grouped by visit, and converted to a list of data
465  references to warps.
466  warpRefList : `list`
467  List of data references to Warps to be coadded.
468  Note: `warpRefList` is just the new name for `tempExpRefList`.
469 
470  Returns
471  -------
472  retStruct : `lsst.pipe.base.Struct`
473  Result struct with components:
474 
475  - ``coaddExposure``: coadded exposure (``Exposure``).
476  - ``nImage``: exposure count image (``Image``).
477  """
478  if selectDataList and warpRefList:
479  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
480  "and which to use is ambiguous. Please pass only one.")
481 
482  skyInfo = self.getSkyInfo(dataRef)
483  if warpRefList is None:
484  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
485  if len(calExpRefList) == 0:
486  self.log.warn("No exposures to coadd")
487  return
488  self.log.info("Coadding %d exposures", len(calExpRefList))
489 
490  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
491 
492  inputData = self.prepareInputs(warpRefList)
493  self.log.info("Found %d %s", len(inputData.tempExpRefList),
494  self.getTempExpDatasetName(self.warpType))
495  if len(inputData.tempExpRefList) == 0:
496  self.log.warn("No coadd temporary exposures found")
497  return
498 
499  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
500 
501  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
502  inputData.weightList, supplementaryData=supplementaryData)
503 
504  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
505  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
506 
507  if self.config.doWrite:
508  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
509  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
510  else:
511  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
512  self.log.info("Persisting %s" % coaddDatasetName)
513  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
514  if self.config.doNImage and retStruct.nImage is not None:
515  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
516 
517  return retStruct
518 
519  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
520  """Interpolate over missing data and mask bright stars.
521 
522  Parameters
523  ----------
524  coaddExposure : `lsst.afw.image.Exposure`
525  The coadded exposure to process.
526  dataRef : `lsst.daf.persistence.ButlerDataRef`
527  Butler data reference for supplementary data.
528  """
529  if self.config.doInterp:
530  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
531  # The variance must be positive; work around for DM-3201.
532  varArray = coaddExposure.variance.array
533  with numpy.errstate(invalid="ignore"):
534  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
535 
536  if self.config.doMaskBrightObjects:
537  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
538 
539  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
540  """Make additional inputs to run() specific to subclasses (Gen2)
541 
542  Duplicates interface of `runDataRef` method
543  Available to be implemented by subclasses only if they need the
544  coadd dataRef for performing preliminary processing before
545  assembling the coadd.
546 
547  Parameters
548  ----------
549  dataRef : `lsst.daf.persistence.ButlerDataRef`
550  Butler data reference for supplementary data.
551  selectDataList : `list` (optional)
552  Optional List of data references to Calexps.
553  warpRefList : `list` (optional)
554  Optional List of data references to Warps.
555  """
556  return pipeBase.Struct()
557 
558  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
559  """Make additional inputs to run() specific to subclasses (Gen3)
560 
561  Duplicates interface of `runQuantum` method.
562  Available to be implemented by subclasses only if they need the
563  coadd dataRef for performing preliminary processing before
564  assembling the coadd.
565 
566  Parameters
567  ----------
568  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
569  Gen3 Butler object for fetching additional data products before
570  running the Task specialized for quantum being processed
571  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
572  Attributes are the names of the connections describing input dataset types.
573  Values are DatasetRefs that task consumes for corresponding dataset type.
574  DataIds are guaranteed to match data objects in ``inputData``.
575  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
576  Attributes are the names of the connections describing output dataset types.
577  Values are DatasetRefs that task is to produce
578  for corresponding dataset type.
579  """
580  return pipeBase.Struct()
581 
582  def getTempExpRefList(self, patchRef, calExpRefList):
583  """Generate list data references corresponding to warped exposures
584  that lie within the patch to be coadded.
585 
586  Parameters
587  ----------
588  patchRef : `dataRef`
589  Data reference for patch.
590  calExpRefList : `list`
591  List of data references for input calexps.
592 
593  Returns
594  -------
595  tempExpRefList : `list`
596  List of Warp/CoaddTempExp data references.
597  """
598  butler = patchRef.getButler()
599  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
600  self.getTempExpDatasetName(self.warpType))
601  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
602  g, groupData.keys) for
603  g in groupData.groups.keys()]
604  return tempExpRefList
605 
606  def prepareInputs(self, refList):
607  """Prepare the input warps for coaddition by measuring the weight for
608  each warp and the scaling for the photometric zero point.
609 
610  Each Warp has its own photometric zeropoint and background variance.
611  Before coadding these Warps together, compute a scale factor to
612  normalize the photometric zeropoint and compute the weight for each Warp.
613 
614  Parameters
615  ----------
616  refList : `list`
617  List of data references to tempExp
618 
619  Returns
620  -------
621  result : `lsst.pipe.base.Struct`
622  Result struct with components:
623 
624  - ``tempExprefList``: `list` of data references to tempExp.
625  - ``weightList``: `list` of weightings.
626  - ``imageScalerList``: `list` of image scalers.
627  """
628  statsCtrl = afwMath.StatisticsControl()
629  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
630  statsCtrl.setNumIter(self.config.clipIter)
631  statsCtrl.setAndMask(self.getBadPixelMask())
632  statsCtrl.setNanSafe(True)
633  # compute tempExpRefList: a list of tempExpRef that actually exist
634  # and weightList: a list of the weight of the associated coadd tempExp
635  # and imageScalerList: a list of scale factors for the associated coadd tempExp
636  tempExpRefList = []
637  weightList = []
638  imageScalerList = []
639  tempExpName = self.getTempExpDatasetName(self.warpType)
640  for tempExpRef in refList:
641  # Gen3's DeferredDatasetHandles are guaranteed to exist and
642  # therefore have no datasetExists() method
643  if not isinstance(tempExpRef, DeferredDatasetHandle):
644  if not tempExpRef.datasetExists(tempExpName):
645  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
646  continue
647 
648  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
649  # Ignore any input warp that is empty of data
650  if numpy.isnan(tempExp.image.array).all():
651  continue
652  maskedImage = tempExp.getMaskedImage()
653  imageScaler = self.scaleZeroPoint.computeImageScaler(
654  exposure=tempExp,
655  dataRef=tempExpRef,
656  )
657  try:
658  imageScaler.scaleMaskedImage(maskedImage)
659  except Exception as e:
660  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
661  continue
662  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
663  afwMath.MEANCLIP, statsCtrl)
664  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
665  weight = 1.0 / float(meanVar)
666  if not numpy.isfinite(weight):
667  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
668  continue
669  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
670 
671  del maskedImage
672  del tempExp
673 
674  tempExpRefList.append(tempExpRef)
675  weightList.append(weight)
676  imageScalerList.append(imageScaler)
677 
678  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
679  imageScalerList=imageScalerList)
680 
681  def prepareStats(self, mask=None):
682  """Prepare the statistics for coadding images.
683 
684  Parameters
685  ----------
686  mask : `int`, optional
687  Bit mask value to exclude from coaddition.
688 
689  Returns
690  -------
691  stats : `lsst.pipe.base.Struct`
692  Statistics structure with the following fields:
693 
694  - ``statsCtrl``: Statistics control object for coadd
695  (`lsst.afw.math.StatisticsControl`)
696  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
697  """
698  if mask is None:
699  mask = self.getBadPixelMask()
700  statsCtrl = afwMath.StatisticsControl()
701  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
702  statsCtrl.setNumIter(self.config.clipIter)
703  statsCtrl.setAndMask(mask)
704  statsCtrl.setNanSafe(True)
705  statsCtrl.setWeighted(True)
706  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
707  for plane, threshold in self.config.maskPropagationThresholds.items():
708  bit = afwImage.Mask.getMaskPlane(plane)
709  statsCtrl.setMaskPropagationThreshold(bit, threshold)
710  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
711  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
712 
713  @pipeBase.timeMethod
714  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
715  altMaskList=None, mask=None, supplementaryData=None):
716  """Assemble a coadd from input warps
717 
718  Assemble the coadd using the provided list of coaddTempExps. Since
719  the full coadd covers a patch (a large area), the assembly is
720  performed over small areas on the image at a time in order to
721  conserve memory usage. Iterate over subregions within the outer
722  bbox of the patch using `assembleSubregion` to stack the corresponding
723  subregions from the coaddTempExps with the statistic specified.
724  Set the edge bits the coadd mask based on the weight map.
725 
726  Parameters
727  ----------
728  skyInfo : `lsst.pipe.base.Struct`
729  Struct with geometric information about the patch.
730  tempExpRefList : `list`
731  List of data references to Warps (previously called CoaddTempExps).
732  imageScalerList : `list`
733  List of image scalers.
734  weightList : `list`
735  List of weights
736  altMaskList : `list`, optional
737  List of alternate masks to use rather than those stored with
738  tempExp.
739  mask : `int`, optional
740  Bit mask value to exclude from coaddition.
741  supplementaryData : lsst.pipe.base.Struct, optional
742  Struct with additional data products needed to assemble coadd.
743  Only used by subclasses that implement `makeSupplementaryData`
744  and override `run`.
745 
746  Returns
747  -------
748  result : `lsst.pipe.base.Struct`
749  Result struct with components:
750 
751  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
752  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
753  - ``warpRefList``: input list of refs to the warps (
754  ``lsst.daf.butler.DeferredDatasetHandle`` or
755  ``lsst.daf.persistence.ButlerDataRef``)
756  (unmodified)
757  - ``imageScalerList``: input list of image scalers (unmodified)
758  - ``weightList``: input list of weights (unmodified)
759  """
760  tempExpName = self.getTempExpDatasetName(self.warpType)
761  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
762  stats = self.prepareStats(mask=mask)
763 
764  if altMaskList is None:
765  altMaskList = [None]*len(tempExpRefList)
766 
767  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
768  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
769  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
770  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
771  coaddMaskedImage = coaddExposure.getMaskedImage()
772  subregionSizeArr = self.config.subregionSize
773  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
774  # if nImage is requested, create a zero one which can be passed to assembleSubregion
775  if self.config.doNImage:
776  nImage = afwImage.ImageU(skyInfo.bbox)
777  else:
778  nImage = None
779  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
780  try:
781  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
782  weightList, altMaskList, stats.flags, stats.ctrl,
783  nImage=nImage)
784  except Exception as e:
785  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
786 
787  self.setInexactPsf(coaddMaskedImage.getMask())
788  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
789  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
790  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
791  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
792  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
793  weightList=weightList)
794 
795  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
796  """Set the metadata for the coadd.
797 
798  This basic implementation sets the filter from the first input.
799 
800  Parameters
801  ----------
802  coaddExposure : `lsst.afw.image.Exposure`
803  The target exposure for the coadd.
804  tempExpRefList : `list`
805  List of data references to tempExp.
806  weightList : `list`
807  List of weights.
808  """
809  assert len(tempExpRefList) == len(weightList), "Length mismatch"
810  tempExpName = self.getTempExpDatasetName(self.warpType)
811  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
812  # (and we need more than just the PropertySet that contains the header), which is not possible
813  # with the current butler (see #2777).
814  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
815 
816  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
817  # Gen 3 API
818  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
819  else:
820  # Gen 2 API. Delete this when Gen 2 retired
821  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
822  for tempExpRef in tempExpRefList]
823  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
824 
825  coaddExposure.setFilter(tempExpList[0].getFilter())
826  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
827  coaddInputs.ccds.reserve(numCcds)
828  coaddInputs.visits.reserve(len(tempExpList))
829 
830  for tempExp, weight in zip(tempExpList, weightList):
831  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
832 
833  if self.config.doUsePsfMatchedPolygons:
834  self.shrinkValidPolygons(coaddInputs)
835 
836  coaddInputs.visits.sort()
837  if self.warpType == "psfMatched":
838  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
839  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
840  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
841  # having the maximum width (sufficient because square)
842  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
843  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
844  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
845  else:
846  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
847  self.config.coaddPsf.makeControl())
848  coaddExposure.setPsf(psf)
849  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
850  coaddExposure.getWcs())
851  coaddExposure.getInfo().setApCorrMap(apCorrMap)
852  if self.config.doAttachTransmissionCurve:
853  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
854  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
855 
856  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
857  altMaskList, statsFlags, statsCtrl, nImage=None):
858  """Assemble the coadd for a sub-region.
859 
860  For each coaddTempExp, check for (and swap in) an alternative mask
861  if one is passed. Remove mask planes listed in
862  `config.removeMaskPlanes`. Finally, stack the actual exposures using
863  `lsst.afw.math.statisticsStack` with the statistic specified by
864  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
865  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
866  an N-sigma clipped mean where N and iterations are specified by
867  statsCtrl. Assign the stacked subregion back to the coadd.
868 
869  Parameters
870  ----------
871  coaddExposure : `lsst.afw.image.Exposure`
872  The target exposure for the coadd.
873  bbox : `lsst.geom.Box`
874  Sub-region to coadd.
875  tempExpRefList : `list`
876  List of data reference to tempExp.
877  imageScalerList : `list`
878  List of image scalers.
879  weightList : `list`
880  List of weights.
881  altMaskList : `list`
882  List of alternate masks to use rather than those stored with
883  tempExp, or None. Each element is dict with keys = mask plane
884  name to which to add the spans.
885  statsFlags : `lsst.afw.math.Property`
886  Property object for statistic for coadd.
887  statsCtrl : `lsst.afw.math.StatisticsControl`
888  Statistics control object for coadd.
889  nImage : `lsst.afw.image.ImageU`, optional
890  Keeps track of exposure count for each pixel.
891  """
892  self.log.debug("Computing coadd over %s", bbox)
893  tempExpName = self.getTempExpDatasetName(self.warpType)
894  coaddExposure.mask.addMaskPlane("REJECTED")
895  coaddExposure.mask.addMaskPlane("CLIPPED")
896  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
897  maskMap = self.setRejectedMaskMapping(statsCtrl)
898  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
899  maskedImageList = []
900  if nImage is not None:
901  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
902  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
903 
904  if isinstance(tempExpRef, DeferredDatasetHandle):
905  # Gen 3 API
906  exposure = tempExpRef.get(parameters={'bbox': bbox})
907  else:
908  # Gen 2 API. Delete this when Gen 2 retired
909  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
910 
911  maskedImage = exposure.getMaskedImage()
912  mask = maskedImage.getMask()
913  if altMask is not None:
914  self.applyAltMaskPlanes(mask, altMask)
915  imageScaler.scaleMaskedImage(maskedImage)
916 
917  # Add 1 for each pixel which is not excluded by the exclude mask.
918  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
919  if nImage is not None:
920  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
921  if self.config.removeMaskPlanes:
922  self.removeMaskPlanes(maskedImage)
923  maskedImageList.append(maskedImage)
924 
925  with self.timer("stack"):
926  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
927  clipped, # also set output to CLIPPED if sigma-clipped
928  maskMap)
929  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
930  if nImage is not None:
931  nImage.assign(subNImage, bbox)
932 
933  def removeMaskPlanes(self, maskedImage):
934  """Unset the mask of an image for mask planes specified in the config.
935 
936  Parameters
937  ----------
938  maskedImage : `lsst.afw.image.MaskedImage`
939  The masked image to be modified.
940  """
941  mask = maskedImage.getMask()
942  for maskPlane in self.config.removeMaskPlanes:
943  try:
944  mask &= ~mask.getPlaneBitMask(maskPlane)
945  except pexExceptions.InvalidParameterError:
946  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
947  maskPlane)
948 
949  @staticmethod
950  def setRejectedMaskMapping(statsCtrl):
951  """Map certain mask planes of the warps to new planes for the coadd.
952 
953  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
954  or CLIPPED, set it to REJECTED on the coadd.
955  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
956  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
957 
958  Parameters
959  ----------
960  statsCtrl : `lsst.afw.math.StatisticsControl`
961  Statistics control object for coadd
962 
963  Returns
964  -------
965  maskMap : `list` of `tuple` of `int`
966  A list of mappings of mask planes of the warped exposures to
967  mask planes of the coadd.
968  """
969  edge = afwImage.Mask.getPlaneBitMask("EDGE")
970  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
971  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
972  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
973  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
974  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
975  (clipped, clipped)]
976  return maskMap
977 
978  def applyAltMaskPlanes(self, mask, altMaskSpans):
979  """Apply in place alt mask formatted as SpanSets to a mask.
980 
981  Parameters
982  ----------
983  mask : `lsst.afw.image.Mask`
984  Original mask.
985  altMaskSpans : `dict`
986  SpanSet lists to apply. Each element contains the new mask
987  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
988  and list of SpanSets to apply to the mask.
989 
990  Returns
991  -------
992  mask : `lsst.afw.image.Mask`
993  Updated mask.
994  """
995  if self.config.doUsePsfMatchedPolygons:
996  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
997  # Clear away any other masks outside the validPolygons. These pixels are no longer
998  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
999  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1000  # This mask-clearing step must only occur *before* applying the new masks below
1001  for spanSet in altMaskSpans['NO_DATA']:
1002  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1003 
1004  for plane, spanSetList in altMaskSpans.items():
1005  maskClipValue = mask.addMaskPlane(plane)
1006  for spanSet in spanSetList:
1007  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1008  return mask
1009 
1010  def shrinkValidPolygons(self, coaddInputs):
1011  """Shrink coaddInputs' ccds' ValidPolygons in place.
1012 
1013  Either modify each ccd's validPolygon in place, or if CoaddInputs
1014  does not have a validPolygon, create one from its bbox.
1015 
1016  Parameters
1017  ----------
1018  coaddInputs : `lsst.afw.image.coaddInputs`
1019  Original mask.
1020 
1021  """
1022  for ccd in coaddInputs.ccds:
1023  polyOrig = ccd.getValidPolygon()
1024  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1025  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1026  if polyOrig:
1027  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1028  else:
1029  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1030  ccd.setValidPolygon(validPolygon)
1031 
1032  def readBrightObjectMasks(self, dataRef):
1033  """Retrieve the bright object masks.
1034 
1035  Returns None on failure.
1036 
1037  Parameters
1038  ----------
1039  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1040  A Butler dataRef.
1041 
1042  Returns
1043  -------
1044  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1045  Bright object mask from the Butler object, or None if it cannot
1046  be retrieved.
1047  """
1048  try:
1049  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1050  except Exception as e:
1051  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1052  return None
1053 
1054  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1055  """Set the bright object masks.
1056 
1057  Parameters
1058  ----------
1059  exposure : `lsst.afw.image.Exposure`
1060  Exposure under consideration.
1061  dataId : `lsst.daf.persistence.dataId`
1062  Data identifier dict for patch.
1063  brightObjectMasks : `lsst.afw.table`
1064  Table of bright objects to mask.
1065  """
1066 
1067  if brightObjectMasks is None:
1068  self.log.warn("Unable to apply bright object mask: none supplied")
1069  return
1070  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1071  mask = exposure.getMaskedImage().getMask()
1072  wcs = exposure.getWcs()
1073  plateScale = wcs.getPixelScale().asArcseconds()
1074 
1075  for rec in brightObjectMasks:
1076  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1077  if rec["type"] == "box":
1078  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1079  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1080  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1081 
1082  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1083  bbox = geom.Box2I(center - halfSize, center + halfSize)
1084 
1085  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1086  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1087  spans = afwGeom.SpanSet(bbox)
1088  elif rec["type"] == "circle":
1089  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1090  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1091  else:
1092  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1093  continue
1094  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1095 
1096  def setInexactPsf(self, mask):
1097  """Set INEXACT_PSF mask plane.
1098 
1099  If any of the input images isn't represented in the coadd (due to
1100  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1101  these pixels.
1102 
1103  Parameters
1104  ----------
1105  mask : `lsst.afw.image.Mask`
1106  Coadded exposure's mask, modified in-place.
1107  """
1108  mask.addMaskPlane("INEXACT_PSF")
1109  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1110  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1111  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1112  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1113  array = mask.getArray()
1114  selected = array & (sensorEdge | clipped | rejected) > 0
1115  array[selected] |= inexactPsf
1116 
1117  @classmethod
1118  def _makeArgumentParser(cls):
1119  """Create an argument parser.
1120  """
1121  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1122  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1123  + cls.ConfigClass().warpType + "Warp",
1124  help="data ID, e.g. --id tract=12345 patch=1,2",
1125  ContainerClass=AssembleCoaddDataIdContainer)
1126  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1127  ContainerClass=SelectDataIdContainer)
1128  return parser
1129 
1130  @staticmethod
1131  def _subBBoxIter(bbox, subregionSize):
1132  """Iterate over subregions of a bbox.
1133 
1134  Parameters
1135  ----------
1136  bbox : `lsst.geom.Box2I`
1137  Bounding box over which to iterate.
1138  subregionSize: `lsst.geom.Extent2I`
1139  Size of sub-bboxes.
1140 
1141  Yields
1142  ------
1143  subBBox : `lsst.geom.Box2I`
1144  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1145  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1146  the edges of ``bbox``, but it will never be empty.
1147  """
1148  if bbox.isEmpty():
1149  raise RuntimeError("bbox %s is empty" % (bbox,))
1150  if subregionSize[0] < 1 or subregionSize[1] < 1:
1151  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1152 
1153  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1154  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1155  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1156  subBBox.clip(bbox)
1157  if subBBox.isEmpty():
1158  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1159  "colShift=%s, rowShift=%s" %
1160  (bbox, subregionSize, colShift, rowShift))
1161  yield subBBox
1162 
1163 
1164 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1165  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1166  """
1167 
1168  def makeDataRefList(self, namespace):
1169  """Make self.refList from self.idList.
1170 
1171  Parameters
1172  ----------
1173  namespace
1174  Results of parsing command-line (with ``butler`` and ``log`` elements).
1175  """
1176  datasetType = namespace.config.coaddName + "Coadd"
1177  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1178 
1179  for dataId in self.idList:
1180  # tract and patch are required
1181  for key in keysCoadd:
1182  if key not in dataId:
1183  raise RuntimeError("--id must include " + key)
1184 
1185  dataRef = namespace.butler.dataRef(
1186  datasetType=datasetType,
1187  dataId=dataId,
1188  )
1189  self.refList.append(dataRef)
1190 
1191 
1192 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1193  """Function to count the number of pixels with a specific mask in a
1194  footprint.
1195 
1196  Find the intersection of mask & footprint. Count all pixels in the mask
1197  that are in the intersection that have bitmask set but do not have
1198  ignoreMask set. Return the count.
1199 
1200  Parameters
1201  ----------
1202  mask : `lsst.afw.image.Mask`
1203  Mask to define intersection region by.
1204  footprint : `lsst.afw.detection.Footprint`
1205  Footprint to define the intersection region by.
1206  bitmask
1207  Specific mask that we wish to count the number of occurances of.
1208  ignoreMask
1209  Pixels to not consider.
1210 
1211  Returns
1212  -------
1213  result : `int`
1214  Count of number of pixels in footprint with specified mask.
1215  """
1216  bbox = footprint.getBBox()
1217  bbox.clip(mask.getBBox(afwImage.PARENT))
1218  fp = afwImage.Mask(bbox)
1219  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1220  footprint.spans.setMask(fp, bitmask)
1221  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1222  (subMask.getArray() & ignoreMask) == 0).sum()
1223 
1224 
1225 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1226  """Configuration parameters for the SafeClipAssembleCoaddTask.
1227  """
1228  assembleMeanCoadd = pexConfig.ConfigurableField(
1229  target=AssembleCoaddTask,
1230  doc="Task to assemble an initial Coadd using the MEAN statistic.",
1231  )
1232  assembleMeanClipCoadd = pexConfig.ConfigurableField(
1233  target=AssembleCoaddTask,
1234  doc="Task to assemble an initial Coadd using the MEANCLIP statistic.",
1235  )
1236  clipDetection = pexConfig.ConfigurableField(
1237  target=SourceDetectionTask,
1238  doc="Detect sources on difference between unclipped and clipped coadd")
1239  minClipFootOverlap = pexConfig.Field(
1240  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1241  dtype=float,
1242  default=0.6
1243  )
1244  minClipFootOverlapSingle = pexConfig.Field(
1245  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1246  "clipped when only one visit overlaps",
1247  dtype=float,
1248  default=0.5
1249  )
1250  minClipFootOverlapDouble = pexConfig.Field(
1251  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1252  "clipped when two visits overlap",
1253  dtype=float,
1254  default=0.45
1255  )
1256  maxClipFootOverlapDouble = pexConfig.Field(
1257  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1258  "considering two visits",
1259  dtype=float,
1260  default=0.15
1261  )
1262  minBigOverlap = pexConfig.Field(
1263  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1264  "when labeling clipped footprints",
1265  dtype=int,
1266  default=100
1267  )
1268 
1269  def setDefaults(self):
1270  """Set default values for clipDetection.
1271 
1272  Notes
1273  -----
1274  The numeric values for these configuration parameters were
1275  empirically determined, future work may further refine them.
1276  """
1277  AssembleCoaddConfig.setDefaults(self)
1278  self.clipDetection.doTempLocalBackground = False
1279  self.clipDetection.reEstimateBackground = False
1280  self.clipDetection.returnOriginalFootprints = False
1281  self.clipDetection.thresholdPolarity = "both"
1282  self.clipDetection.thresholdValue = 2
1283  self.clipDetection.nSigmaToGrow = 2
1284  self.clipDetection.minPixels = 4
1285  self.clipDetection.isotropicGrow = True
1286  self.clipDetection.thresholdType = "pixel_stdev"
1287  self.sigmaClip = 1.5
1288  self.clipIter = 3
1289  self.statistic = "MEAN"
1290  self.assembleMeanCoadd.statistic = 'MEAN'
1291  self.assembleMeanClipCoadd.statistic = 'MEANCLIP'
1292  self.assembleMeanCoadd.doWrite = False
1293  self.assembleMeanClipCoadd.doWrite = False
1294 
1295  def validate(self):
1296  if self.doSigmaClip:
1297  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1298  "Ignoring doSigmaClip.")
1299  self.doSigmaClip = False
1300  if self.statistic != "MEAN":
1301  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1302  "(%s chosen). Please set statistic to MEAN."
1303  % (self.statistic))
1304  AssembleCoaddTask.ConfigClass.validate(self)
1305 
1306 
1307 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1308  """Assemble a coadded image from a set of coadded temporary exposures,
1309  being careful to clip & flag areas with potential artifacts.
1310 
1311  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1312  we clip outliers). The problem with doing this is that when computing the
1313  coadd PSF at a given location, individual visit PSFs from visits with
1314  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1315  In this task, we correct for this behavior by creating a new
1316  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1317  coaddTempExps and the final coadd where
1318 
1319  i. difference imaging suggests that there is an outlier and
1320  ii. this outlier appears on only one or two images.
1321 
1322  Such regions will not contribute to the final coadd. Furthermore, any
1323  routine to determine the coadd PSF can now be cognizant of clipped regions.
1324  Note that the algorithm implemented by this task is preliminary and works
1325  correctly for HSC data. Parameter modifications and or considerable
1326  redesigning of the algorithm is likley required for other surveys.
1327 
1328  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1329  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1330  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1331  if you wish.
1332 
1333  Notes
1334  -----
1335  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1336  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1337  see `baseDebug` for more about ``debug.py`` files.
1338  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1339  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1340  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1341  for further information.
1342 
1343  Examples
1344  --------
1345  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1346  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1347  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1348 
1349  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1350  and filter to be coadded (specified using
1351  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1352  along with a list of coaddTempExps to attempt to coadd (specified using
1353  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1354  Only the coaddTempExps that cover the specified tract and patch will be
1355  coadded. A list of the available optional arguments can be obtained by
1356  calling assembleCoadd.py with the --help command line argument:
1357 
1358  .. code-block:: none
1359 
1360  assembleCoadd.py --help
1361 
1362  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1363  context of multi-band processing, we will generate the HSC-I & -R band
1364  coadds from HSC engineering test data provided in the ci_hsc package.
1365  To begin, assuming that the lsst stack has been already set up, we must
1366  set up the obs_subaru and ci_hsc packages. This defines the environment
1367  variable $CI_HSC_DIR and points at the location of the package. The raw
1368  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1369  the coadds, we must first
1370 
1371  - ``processCcd``
1372  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1373  - ``makeSkyMap``
1374  create a skymap that covers the area of the sky present in the raw exposures
1375  - ``makeCoaddTempExp``
1376  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1377 
1378  We can perform all of these steps by running
1379 
1380  .. code-block:: none
1381 
1382  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1383 
1384  This will produce warped coaddTempExps for each visit. To coadd the
1385  warped data, we call ``assembleCoadd.py`` as follows:
1386 
1387  .. code-block:: none
1388 
1389  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1390  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1391  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1392  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1393  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1394  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1395  --selectId visit=903988 ccd=24
1396 
1397  This will process the HSC-I band data. The results are written in
1398  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1399 
1400  You may also choose to run:
1401 
1402  .. code-block:: none
1403 
1404  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1405  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1406  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1407  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1408  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1409  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1410  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1411  --selectId visit=903346 ccd=12
1412 
1413  to generate the coadd for the HSC-R band if you are interested in following
1414  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1415  """
1416  ConfigClass = SafeClipAssembleCoaddConfig
1417  _DefaultName = "safeClipAssembleCoadd"
1418 
1419  def __init__(self, *args, **kwargs):
1420  AssembleCoaddTask.__init__(self, *args, **kwargs)
1421  schema = afwTable.SourceTable.makeMinimalSchema()
1422  self.makeSubtask("clipDetection", schema=schema)
1423  self.makeSubtask("assembleMeanClipCoadd")
1424  self.makeSubtask("assembleMeanCoadd")
1425 
1426  @utils.inheritDoc(AssembleCoaddTask)
1427  @pipeBase.timeMethod
1428  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1429  """Assemble the coadd for a region.
1430 
1431  Compute the difference of coadds created with and without outlier
1432  rejection to identify coadd pixels that have outlier values in some
1433  individual visits.
1434  Detect clipped regions on the difference image and mark these regions
1435  on the one or two individual coaddTempExps where they occur if there
1436  is significant overlap between the clipped region and a source. This
1437  leaves us with a set of footprints from the difference image that have
1438  been identified as having occured on just one or two individual visits.
1439  However, these footprints were generated from a difference image. It
1440  is conceivable for a large diffuse source to have become broken up
1441  into multiple footprints acrosss the coadd difference in this process.
1442  Determine the clipped region from all overlapping footprints from the
1443  detected sources in each visit - these are big footprints.
1444  Combine the small and big clipped footprints and mark them on a new
1445  bad mask plane.
1446  Generate the coadd using `AssembleCoaddTask.run` without outlier
1447  removal. Clipped footprints will no longer make it into the coadd
1448  because they are marked in the new bad mask plane.
1449 
1450  Notes
1451  -----
1452  args and kwargs are passed but ignored in order to match the call
1453  signature expected by the parent task.
1454  """
1455  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1456  mask = exp.getMaskedImage().getMask()
1457  mask.addMaskPlane("CLIPPED")
1458 
1459  result = self.detectClip(exp, tempExpRefList)
1460 
1461  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1462 
1463  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1464  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1465  # Append big footprints from individual Warps to result.clipSpans
1466  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1467  result.detectionFootprints, maskClipValue, maskDetValue,
1468  exp.getBBox())
1469  # Create mask of the current clipped footprints
1470  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1471  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1472 
1473  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1474  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1475  maskClip |= maskClipBig
1476 
1477  # Assemble coadd from base class, but ignoring CLIPPED pixels
1478  badMaskPlanes = self.config.badMaskPlanes[:]
1479  badMaskPlanes.append("CLIPPED")
1480  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1481  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1482  result.clipSpans, mask=badPixelMask)
1483 
1484  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1485  """Return an exposure that contains the difference between unclipped
1486  and clipped coadds.
1487 
1488  Generate a difference image between clipped and unclipped coadds.
1489  Compute the difference image by subtracting an outlier-clipped coadd
1490  from an outlier-unclipped coadd. Return the difference image.
1491 
1492  Parameters
1493  ----------
1494  skyInfo : `lsst.pipe.base.Struct`
1495  Patch geometry information, from getSkyInfo
1496  tempExpRefList : `list`
1497  List of data reference to tempExp
1498  imageScalerList : `list`
1499  List of image scalers
1500  weightList : `list`
1501  List of weights
1502 
1503  Returns
1504  -------
1505  exp : `lsst.afw.image.Exposure`
1506  Difference image of unclipped and clipped coadd wrapped in an Exposure
1507  """
1508  coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList,
1509  imageScalerList, weightList).coaddExposure
1510 
1511  coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList,
1512  imageScalerList, weightList).coaddExposure
1513 
1514  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1515  coaddDiff -= coaddClip.getMaskedImage()
1516  exp = afwImage.ExposureF(coaddDiff)
1517  exp.setPsf(coaddMean.getPsf())
1518  return exp
1519 
1520  def detectClip(self, exp, tempExpRefList):
1521  """Detect clipped regions on an exposure and set the mask on the
1522  individual tempExp masks.
1523 
1524  Detect footprints in the difference image after smoothing the
1525  difference image with a Gaussian kernal. Identify footprints that
1526  overlap with one or two input ``coaddTempExps`` by comparing the
1527  computed overlap fraction to thresholds set in the config. A different
1528  threshold is applied depending on the number of overlapping visits
1529  (restricted to one or two). If the overlap exceeds the thresholds,
1530  the footprint is considered "CLIPPED" and is marked as such on the
1531  coaddTempExp. Return a struct with the clipped footprints, the indices
1532  of the ``coaddTempExps`` that end up overlapping with the clipped
1533  footprints, and a list of new masks for the ``coaddTempExps``.
1534 
1535  Parameters
1536  ----------
1537  exp : `lsst.afw.image.Exposure`
1538  Exposure to run detection on.
1539  tempExpRefList : `list`
1540  List of data reference to tempExp.
1541 
1542  Returns
1543  -------
1544  result : `lsst.pipe.base.Struct`
1545  Result struct with components:
1546 
1547  - ``clipFootprints``: list of clipped footprints.
1548  - ``clipIndices``: indices for each ``clippedFootprint`` in
1549  ``tempExpRefList``.
1550  - ``clipSpans``: List of dictionaries containing spanSet lists
1551  to clip. Each element contains the new maskplane name
1552  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1553  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1554  compressed into footprints.
1555  """
1556  mask = exp.getMaskedImage().getMask()
1557  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1558  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1559  # Merge positive and negative together footprints together
1560  fpSet.positive.merge(fpSet.negative)
1561  footprints = fpSet.positive
1562  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1563  ignoreMask = self.getBadPixelMask()
1564 
1565  clipFootprints = []
1566  clipIndices = []
1567  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1568 
1569  # for use by detectClipBig
1570  visitDetectionFootprints = []
1571 
1572  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1573  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1574  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1575 
1576  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1577  for i, warpRef in enumerate(tempExpRefList):
1578  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1579  immediate=True).getMaskedImage().getMask()
1580  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1581  afwImage.PARENT, True)
1582  maskVisitDet &= maskDetValue
1583  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1584  visitDetectionFootprints.append(visitFootprints)
1585 
1586  for j, footprint in enumerate(footprints.getFootprints()):
1587  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1588  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1589 
1590  # build a list of clipped spans for each visit
1591  for j, footprint in enumerate(footprints.getFootprints()):
1592  nPixel = footprint.getArea()
1593  overlap = [] # hold the overlap with each visit
1594  indexList = [] # index of visit in global list
1595  for i in range(len(tempExpRefList)):
1596  ignore = ignoreArr[i, j]
1597  overlapDet = overlapDetArr[i, j]
1598  totPixel = nPixel - ignore
1599 
1600  # If we have more bad pixels than detection skip
1601  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1602  continue
1603  overlap.append(overlapDet/float(totPixel))
1604  indexList.append(i)
1605 
1606  overlap = numpy.array(overlap)
1607  if not len(overlap):
1608  continue
1609 
1610  keep = False # Should this footprint be marked as clipped?
1611  keepIndex = [] # Which tempExps does the clipped footprint belong to
1612 
1613  # If footprint only has one overlap use a lower threshold
1614  if len(overlap) == 1:
1615  if overlap[0] > self.config.minClipFootOverlapSingle:
1616  keep = True
1617  keepIndex = [0]
1618  else:
1619  # This is the general case where only visit should be clipped
1620  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1621  if len(clipIndex) == 1:
1622  keep = True
1623  keepIndex = [clipIndex[0]]
1624 
1625  # Test if there are clipped objects that overlap two different visits
1626  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1627  if len(clipIndex) == 2 and len(overlap) > 3:
1628  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1629  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1630  keep = True
1631  keepIndex = clipIndex
1632 
1633  if not keep:
1634  continue
1635 
1636  for index in keepIndex:
1637  globalIndex = indexList[index]
1638  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1639 
1640  clipIndices.append(numpy.array(indexList)[keepIndex])
1641  clipFootprints.append(footprint)
1642 
1643  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1644  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1645 
1646  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1647  maskClipValue, maskDetValue, coaddBBox):
1648  """Return individual warp footprints for large artifacts and append
1649  them to ``clipList`` in place.
1650 
1651  Identify big footprints composed of many sources in the coadd
1652  difference that may have originated in a large diffuse source in the
1653  coadd. We do this by indentifying all clipped footprints that overlap
1654  significantly with each source in all the coaddTempExps.
1655 
1656  Parameters
1657  ----------
1658  clipList : `list`
1659  List of alt mask SpanSets with clipping information. Modified.
1660  clipFootprints : `list`
1661  List of clipped footprints.
1662  clipIndices : `list`
1663  List of which entries in tempExpClipList each footprint belongs to.
1664  maskClipValue
1665  Mask value of clipped pixels.
1666  maskDetValue
1667  Mask value of detected pixels.
1668  coaddBBox : `lsst.geom.Box`
1669  BBox of the coadd and warps.
1670 
1671  Returns
1672  -------
1673  bigFootprintsCoadd : `list`
1674  List of big footprints
1675  """
1676  bigFootprintsCoadd = []
1677  ignoreMask = self.getBadPixelMask()
1678  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1679  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1680  for footprint in visitFootprints.getFootprints():
1681  footprint.spans.setMask(maskVisitDet, maskDetValue)
1682 
1683  # build a mask of clipped footprints that are in this visit
1684  clippedFootprintsVisit = []
1685  for foot, clipIndex in zip(clipFootprints, clipIndices):
1686  if index not in clipIndex:
1687  continue
1688  clippedFootprintsVisit.append(foot)
1689  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1690  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1691 
1692  bigFootprintsVisit = []
1693  for foot in visitFootprints.getFootprints():
1694  if foot.getArea() < self.config.minBigOverlap:
1695  continue
1696  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1697  if nCount > self.config.minBigOverlap:
1698  bigFootprintsVisit.append(foot)
1699  bigFootprintsCoadd.append(foot)
1700 
1701  for footprint in bigFootprintsVisit:
1702  clippedSpans["CLIPPED"].append(footprint.spans)
1703 
1704  return bigFootprintsCoadd
1705 
1706 
1708  psfMatchedWarps = pipeBase.connectionTypes.Input(
1709  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1710  "Only PSF-Matched Warps make sense for image subtraction. "
1711  "Therefore, they must be an additional declared input."),
1712  name="{inputCoaddName}Coadd_psfMatchedWarp",
1713  storageClass="ExposureF",
1714  dimensions=("tract", "patch", "skymap", "visit"),
1715  deferLoad=True,
1716  multiple=True
1717  )
1718  templateCoadd = pipeBase.connectionTypes.Output(
1719  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1720  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1721  name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1722  storageClass="ExposureF",
1723  dimensions=("tract", "patch", "skymap", "abstract_filter"),
1724  )
1725 
1726  def __init__(self, *, config=None):
1727  super().__init__(config=config)
1728  if not config.assembleStaticSkyModel.doWrite:
1729  self.outputs.remove("templateCoadd")
1730  config.validate()
1731 
1732 
1733 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1734  pipelineConnections=CompareWarpAssembleCoaddConnections):
1735  assembleStaticSkyModel = pexConfig.ConfigurableField(
1736  target=AssembleCoaddTask,
1737  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1738  " naive/first-iteration model of the static sky.",
1739  )
1740  detect = pexConfig.ConfigurableField(
1741  target=SourceDetectionTask,
1742  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1743  )
1744  detectTemplate = pexConfig.ConfigurableField(
1745  target=SourceDetectionTask,
1746  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1747  )
1748  maskStreaks = pexConfig.ConfigurableField(
1749  target=MaskStreaksTask,
1750  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1751  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1752  "streakMaskName"
1753  )
1754  streakMaskName = pexConfig.Field(
1755  dtype=str,
1756  default="STREAK",
1757  doc="Name of mask bit used for streaks"
1758  )
1759  maxNumEpochs = pexConfig.Field(
1760  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1761  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1762  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1763  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1764  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1765  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1766  "than transient and not masked.",
1767  dtype=int,
1768  default=2
1769  )
1770  maxFractionEpochsLow = pexConfig.RangeField(
1771  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1772  "Effective maxNumEpochs = "
1773  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1774  dtype=float,
1775  default=0.4,
1776  min=0., max=1.,
1777  )
1778  maxFractionEpochsHigh = pexConfig.RangeField(
1779  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1780  "Effective maxNumEpochs = "
1781  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1782  dtype=float,
1783  default=0.03,
1784  min=0., max=1.,
1785  )
1786  spatialThreshold = pexConfig.RangeField(
1787  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1788  "temporal criteria. If 0, clip all. If 1, clip none.",
1789  dtype=float,
1790  default=0.5,
1791  min=0., max=1.,
1792  inclusiveMin=True, inclusiveMax=True
1793  )
1794  doScaleWarpVariance = pexConfig.Field(
1795  doc="Rescale Warp variance plane using empirical noise?",
1796  dtype=bool,
1797  default=True,
1798  )
1799  scaleWarpVariance = pexConfig.ConfigurableField(
1800  target=ScaleVarianceTask,
1801  doc="Rescale variance on warps",
1802  )
1803  doPreserveContainedBySource = pexConfig.Field(
1804  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1805  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1806  dtype=bool,
1807  default=True,
1808  )
1809  doPrefilterArtifacts = pexConfig.Field(
1810  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1811  "because they will be excluded anyway. This prevents them from contributing "
1812  "to the outlier epoch count image and potentially being labeled as persistant."
1813  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1814  dtype=bool,
1815  default=True
1816  )
1817  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1818  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1819  dtype=str,
1820  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1821  )
1822  prefilterArtifactsRatio = pexConfig.Field(
1823  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1824  dtype=float,
1825  default=0.05
1826  )
1827  doFilterMorphological = pexConfig.Field(
1828  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1829  "be streaks.",
1830  dtype=bool,
1831  default=False
1832  )
1833 
1834  def setDefaults(self):
1835  AssembleCoaddConfig.setDefaults(self)
1836  self.statistic = 'MEAN'
1838 
1839  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1840  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1841  if "EDGE" in self.badMaskPlanes:
1842  self.badMaskPlanes.remove('EDGE')
1843  self.removeMaskPlanes.append('EDGE')
1844  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1845  self.assembleStaticSkyModel.warpType = 'psfMatched'
1846  self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1847  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1848  self.assembleStaticSkyModel.sigmaClip = 2.5
1849  self.assembleStaticSkyModel.clipIter = 3
1850  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1851  self.assembleStaticSkyModel.doWrite = False
1852  self.detect.doTempLocalBackground = False
1853  self.detect.reEstimateBackground = False
1854  self.detect.returnOriginalFootprints = False
1855  self.detect.thresholdPolarity = "both"
1856  self.detect.thresholdValue = 5
1857  self.detect.minPixels = 4
1858  self.detect.isotropicGrow = True
1859  self.detect.thresholdType = "pixel_stdev"
1860  self.detect.nSigmaToGrow = 0.4
1861  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1862  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1863  self.detectTemplate.nSigmaToGrow = 2.4
1864  self.detectTemplate.doTempLocalBackground = False
1865  self.detectTemplate.reEstimateBackground = False
1866  self.detectTemplate.returnOriginalFootprints = False
1867 
1868  def validate(self):
1869  super().validate()
1870  if self.assembleStaticSkyModel.doNImage:
1871  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1872  "Please set assembleStaticSkyModel.doNImage=False")
1873 
1874  if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1875  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1876  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1877  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1878  "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1879 
1880 
1881 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1882  """Assemble a compareWarp coadded image from a set of warps
1883  by masking artifacts detected by comparing PSF-matched warps.
1884 
1885  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1886  we clip outliers). The problem with doing this is that when computing the
1887  coadd PSF at a given location, individual visit PSFs from visits with
1888  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1889  In this task, we correct for this behavior by creating a new badMaskPlane
1890  'CLIPPED' which marks pixels in the individual warps suspected to contain
1891  an artifact. We populate this plane on the input warps by comparing
1892  PSF-matched warps with a PSF-matched median coadd which serves as a
1893  model of the static sky. Any group of pixels that deviates from the
1894  PSF-matched template coadd by more than config.detect.threshold sigma,
1895  is an artifact candidate. The candidates are then filtered to remove
1896  variable sources and sources that are difficult to subtract such as
1897  bright stars. This filter is configured using the config parameters
1898  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1899  the maximum fraction of epochs that the deviation can appear in and still
1900  be considered an artifact. The spatialThreshold is the maximum fraction of
1901  pixels in the footprint of the deviation that appear in other epochs
1902  (where other epochs is defined by the temporalThreshold). If the deviant
1903  region meets this criteria of having a significant percentage of pixels
1904  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1905  set in the mask. These regions will not contribute to the final coadd.
1906  Furthermore, any routine to determine the coadd PSF can now be cognizant
1907  of clipped regions. Note that the algorithm implemented by this task is
1908  preliminary and works correctly for HSC data. Parameter modifications and
1909  or considerable redesigning of the algorithm is likley required for other
1910  surveys.
1911 
1912  ``CompareWarpAssembleCoaddTask`` sub-classes
1913  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1914  as a subtask to generate the TemplateCoadd (the model of the static sky).
1915 
1916  Notes
1917  -----
1918  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1919  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1920  ``baseDebug`` for more about ``debug.py`` files.
1921 
1922  This task supports the following debug variables:
1923 
1924  - ``saveCountIm``
1925  If True then save the Epoch Count Image as a fits file in the `figPath`
1926  - ``figPath``
1927  Path to save the debug fits images and figures
1928 
1929  For example, put something like:
1930 
1931  .. code-block:: python
1932 
1933  import lsstDebug
1934  def DebugInfo(name):
1935  di = lsstDebug.getInfo(name)
1936  if name == "lsst.pipe.tasks.assembleCoadd":
1937  di.saveCountIm = True
1938  di.figPath = "/desired/path/to/debugging/output/images"
1939  return di
1940  lsstDebug.Info = DebugInfo
1941 
1942  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1943  ``--debug`` flag. Some subtasks may have their own debug variables;
1944  see individual Task documentation.
1945 
1946  Examples
1947  --------
1948  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1949  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1950  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1951  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1952  and filter to be coadded (specified using
1953  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1954  along with a list of coaddTempExps to attempt to coadd (specified using
1955  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1956  Only the warps that cover the specified tract and patch will be coadded.
1957  A list of the available optional arguments can be obtained by calling
1958  ``assembleCoadd.py`` with the ``--help`` command line argument:
1959 
1960  .. code-block:: none
1961 
1962  assembleCoadd.py --help
1963 
1964  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1965  context of multi-band processing, we will generate the HSC-I & -R band
1966  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1967  To begin, assuming that the lsst stack has been already set up, we must
1968  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1969  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1970  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1971  directory. To begin assembling the coadds, we must first
1972 
1973  - processCcd
1974  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1975  - makeSkyMap
1976  create a skymap that covers the area of the sky present in the raw exposures
1977  - makeCoaddTempExp
1978  warp the individual calibrated exposures to the tangent plane of the coadd
1979 
1980  We can perform all of these steps by running
1981 
1982  .. code-block:: none
1983 
1984  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1985 
1986  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1987  warped data, we call ``assembleCoadd.py`` as follows:
1988 
1989  .. code-block:: none
1990 
1991  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1992  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1993  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1994  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1995  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1996  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1997  --selectId visit=903988 ccd=24
1998 
1999  This will process the HSC-I band data. The results are written in
2000  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2001  """
2002  ConfigClass = CompareWarpAssembleCoaddConfig
2003  _DefaultName = "compareWarpAssembleCoadd"
2004 
2005  def __init__(self, *args, **kwargs):
2006  AssembleCoaddTask.__init__(self, *args, **kwargs)
2007  self.makeSubtask("assembleStaticSkyModel")
2008  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2009  self.makeSubtask("detect", schema=detectionSchema)
2010  if self.config.doPreserveContainedBySource:
2011  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2012  if self.config.doScaleWarpVariance:
2013  self.makeSubtask("scaleWarpVariance")
2014  if self.config.doFilterMorphological:
2015  self.makeSubtask("maskStreaks")
2016 
2017  @utils.inheritDoc(AssembleCoaddTask)
2018  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2019  """
2020  Generate a templateCoadd to use as a naive model of static sky to
2021  subtract from PSF-Matched warps.
2022 
2023  Returns
2024  -------
2025  result : `lsst.pipe.base.Struct`
2026  Result struct with components:
2027 
2028  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2029  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2030  """
2031  # Ensure that psfMatchedWarps are used as input warps for template generation
2032  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2033  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2034 
2035  # Because subtasks don't have connections we have to make one.
2036  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2037  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2038  if self.config.assembleStaticSkyModel.doWrite:
2039  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2040  # Remove template coadd from both subtask's and main tasks outputs,
2041  # because it is handled by the subtask as `coaddExposure`
2042  del outputRefs.templateCoadd
2043  del staticSkyModelOutputRefs.templateCoadd
2044 
2045  # A PSF-Matched nImage does not exist as a dataset type
2046  if 'nImage' in staticSkyModelOutputRefs.keys():
2047  del staticSkyModelOutputRefs.nImage
2048 
2049  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2050  staticSkyModelOutputRefs)
2051  if templateCoadd is None:
2052  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2053 
2054  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2055  nImage=templateCoadd.nImage,
2056  warpRefList=templateCoadd.warpRefList,
2057  imageScalerList=templateCoadd.imageScalerList,
2058  weightList=templateCoadd.weightList)
2059 
2060  @utils.inheritDoc(AssembleCoaddTask)
2061  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2062  """
2063  Generate a templateCoadd to use as a naive model of static sky to
2064  subtract from PSF-Matched warps.
2065 
2066  Returns
2067  -------
2068  result : `lsst.pipe.base.Struct`
2069  Result struct with components:
2070 
2071  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2072  - ``nImage``: N Image (``lsst.afw.image.Image``)
2073  """
2074  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2075  if templateCoadd is None:
2076  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2077 
2078  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2079  nImage=templateCoadd.nImage,
2080  warpRefList=templateCoadd.warpRefList,
2081  imageScalerList=templateCoadd.imageScalerList,
2082  weightList=templateCoadd.weightList)
2083 
2084  def _noTemplateMessage(self, warpType):
2085  warpName = (warpType[0].upper() + warpType[1:])
2086  message = """No %(warpName)s warps were found to build the template coadd which is
2087  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2088  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2089  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2090 
2091  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2092  another algorithm like:
2093 
2094  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2095  config.assemble.retarget(SafeClipAssembleCoaddTask)
2096  """ % {"warpName": warpName}
2097  return message
2098 
2099  @utils.inheritDoc(AssembleCoaddTask)
2100  @pipeBase.timeMethod
2101  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2102  supplementaryData, *args, **kwargs):
2103  """Assemble the coadd.
2104 
2105  Find artifacts and apply them to the warps' masks creating a list of
2106  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2107  plane. Then pass these alternative masks to the base class's `run`
2108  method.
2109 
2110  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2111  that must contain a ``templateCoadd`` that serves as the
2112  model of the static sky.
2113  """
2114 
2115  # Check and match the order of the supplementaryData
2116  # (PSF-matched) inputs to the order of the direct inputs,
2117  # so that the artifact mask is applied to the right warp
2118  dataIds = [ref.dataId for ref in tempExpRefList]
2119  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2120 
2121  if dataIds != psfMatchedDataIds:
2122  self.log.info("Reordering and or/padding PSF-matched visit input list")
2123  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2124  psfMatchedDataIds, dataIds)
2125  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2126  psfMatchedDataIds, dataIds)
2127 
2128  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2129  spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2130  supplementaryData.warpRefList,
2131  supplementaryData.imageScalerList)
2132 
2133  badMaskPlanes = self.config.badMaskPlanes[:]
2134  badMaskPlanes.append("CLIPPED")
2135  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2136 
2137  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2138  spanSetMaskList, mask=badPixelMask)
2139 
2140  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2141  # Psf-Matching moves the real edge inwards
2142  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2143  return result
2144 
2145  def applyAltEdgeMask(self, mask, altMaskList):
2146  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2147 
2148  Parameters
2149  ----------
2150  mask : `lsst.afw.image.Mask`
2151  Original mask.
2152  altMaskList : `list`
2153  List of Dicts containing ``spanSet`` lists.
2154  Each element contains the new mask plane name (e.g. "CLIPPED
2155  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2156  the mask.
2157  """
2158  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2159  for visitMask in altMaskList:
2160  if "EDGE" in visitMask:
2161  for spanSet in visitMask['EDGE']:
2162  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2163 
2164  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2165  """Find artifacts.
2166 
2167  Loop through warps twice. The first loop builds a map with the count
2168  of how many epochs each pixel deviates from the templateCoadd by more
2169  than ``config.chiThreshold`` sigma. The second loop takes each
2170  difference image and filters the artifacts detected in each using
2171  count map to filter out variable sources and sources that are
2172  difficult to subtract cleanly.
2173 
2174  Parameters
2175  ----------
2176  templateCoadd : `lsst.afw.image.Exposure`
2177  Exposure to serve as model of static sky.
2178  tempExpRefList : `list`
2179  List of data references to warps.
2180  imageScalerList : `list`
2181  List of image scalers.
2182 
2183  Returns
2184  -------
2185  altMasks : `list`
2186  List of dicts containing information about CLIPPED
2187  (i.e., artifacts), NO_DATA, and EDGE pixels.
2188  """
2189 
2190  self.log.debug("Generating Count Image, and mask lists.")
2191  coaddBBox = templateCoadd.getBBox()
2192  slateIm = afwImage.ImageU(coaddBBox)
2193  epochCountImage = afwImage.ImageU(coaddBBox)
2194  nImage = afwImage.ImageU(coaddBBox)
2195  spanSetArtifactList = []
2196  spanSetNoDataMaskList = []
2197  spanSetEdgeList = []
2198  spanSetBadMorphoList = []
2199  badPixelMask = self.getBadPixelMask()
2200 
2201  # mask of the warp diffs should = that of only the warp
2202  templateCoadd.mask.clearAllMaskPlanes()
2203 
2204  if self.config.doPreserveContainedBySource:
2205  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2206  else:
2207  templateFootprints = None
2208 
2209  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2210  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2211  if warpDiffExp is not None:
2212  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2213  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2214  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2215  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2216  fpSet.positive.merge(fpSet.negative)
2217  footprints = fpSet.positive
2218  slateIm.set(0)
2219  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2220 
2221  # Remove artifacts due to defects before they contribute to the epochCountImage
2222  if self.config.doPrefilterArtifacts:
2223  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2224 
2225  # Clear mask before adding prefiltered spanSets
2226  self.detect.clearMask(warpDiffExp.mask)
2227  for spans in spanSetList:
2228  spans.setImage(slateIm, 1, doClip=True)
2229  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2230  epochCountImage += slateIm
2231 
2232  if self.config.doFilterMorphological:
2233  maskName = self.config.streakMaskName
2234  _ = self.maskStreaks.run(warpDiffExp)
2235  streakMask = warpDiffExp.mask
2236  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2237  streakMask.getPlaneBitMask(maskName)).split()
2238 
2239  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2240  # undergo a second convolution. Pixels with data in the direct warp
2241  # but not in the PSF-matched warp will not have their artifacts detected.
2242  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2243  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2244  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2245  nansMask.setXY0(warpDiffExp.getXY0())
2246  edgeMask = warpDiffExp.mask
2247  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2248  edgeMask.getPlaneBitMask("EDGE")).split()
2249  else:
2250  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2251  # In this case, mask the whole epoch
2252  nansMask = afwImage.MaskX(coaddBBox, 1)
2253  spanSetList = []
2254  spanSetEdgeMask = []
2255  spanSetStreak = []
2256 
2257  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2258 
2259  spanSetNoDataMaskList.append(spanSetNoDataMask)
2260  spanSetArtifactList.append(spanSetList)
2261  spanSetEdgeList.append(spanSetEdgeMask)
2262  if self.config.doFilterMorphological:
2263  spanSetBadMorphoList.append(spanSetStreak)
2264 
2265  if lsstDebug.Info(__name__).saveCountIm:
2266  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2267  epochCountImage.writeFits(path)
2268 
2269  for i, spanSetList in enumerate(spanSetArtifactList):
2270  if spanSetList:
2271  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2272  templateFootprints)
2273  spanSetArtifactList[i] = filteredSpanSetList
2274  if self.config.doFilterMorphological:
2275  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2276 
2277  altMasks = []
2278  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2279  altMasks.append({'CLIPPED': artifacts,
2280  'NO_DATA': noData,
2281  'EDGE': edge})
2282  return altMasks
2283 
2284  def prefilterArtifacts(self, spanSetList, exp):
2285  """Remove artifact candidates covered by bad mask plane.
2286 
2287  Any future editing of the candidate list that does not depend on
2288  temporal information should go in this method.
2289 
2290  Parameters
2291  ----------
2292  spanSetList : `list`
2293  List of SpanSets representing artifact candidates.
2294  exp : `lsst.afw.image.Exposure`
2295  Exposure containing mask planes used to prefilter.
2296 
2297  Returns
2298  -------
2299  returnSpanSetList : `list`
2300  List of SpanSets with artifacts.
2301  """
2302  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2303  goodArr = (exp.mask.array & badPixelMask) == 0
2304  returnSpanSetList = []
2305  bbox = exp.getBBox()
2306  x0, y0 = exp.getXY0()
2307  for i, span in enumerate(spanSetList):
2308  y, x = span.clippedTo(bbox).indices()
2309  yIndexLocal = numpy.array(y) - y0
2310  xIndexLocal = numpy.array(x) - x0
2311  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2312  if goodRatio > self.config.prefilterArtifactsRatio:
2313  returnSpanSetList.append(span)
2314  return returnSpanSetList
2315 
2316  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2317  """Filter artifact candidates.
2318 
2319  Parameters
2320  ----------
2321  spanSetList : `list`
2322  List of SpanSets representing artifact candidates.
2323  epochCountImage : `lsst.afw.image.Image`
2324  Image of accumulated number of warpDiff detections.
2325  nImage : `lsst.afw.image.Image`
2326  Image of the accumulated number of total epochs contributing.
2327 
2328  Returns
2329  -------
2330  maskSpanSetList : `list`
2331  List of SpanSets with artifacts.
2332  """
2333 
2334  maskSpanSetList = []
2335  x0, y0 = epochCountImage.getXY0()
2336  for i, span in enumerate(spanSetList):
2337  y, x = span.indices()
2338  yIdxLocal = [y1 - y0 for y1 in y]
2339  xIdxLocal = [x1 - x0 for x1 in x]
2340  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2341  totalN = nImage.array[yIdxLocal, xIdxLocal]
2342 
2343  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2344  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2345  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2346  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2347  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2348  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2349  & (outlierN <= effectiveMaxNumEpochs))
2350  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2351  if percentBelowThreshold > self.config.spatialThreshold:
2352  maskSpanSetList.append(span)
2353 
2354  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2355  # If a candidate is contained by a footprint on the template coadd, do not clip
2356  filteredMaskSpanSetList = []
2357  for span in maskSpanSetList:
2358  doKeep = True
2359  for footprint in footprintsToExclude.positive.getFootprints():
2360  if footprint.spans.contains(span):
2361  doKeep = False
2362  break
2363  if doKeep:
2364  filteredMaskSpanSetList.append(span)
2365  maskSpanSetList = filteredMaskSpanSetList
2366 
2367  return maskSpanSetList
2368 
2369  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2370  """Fetch a warp from the butler and return a warpDiff.
2371 
2372  Parameters
2373  ----------
2374  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2375  Butler dataRef for the warp.
2376  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2377  An image scaler object.
2378  templateCoadd : `lsst.afw.image.Exposure`
2379  Exposure to be substracted from the scaled warp.
2380 
2381  Returns
2382  -------
2383  warp : `lsst.afw.image.Exposure`
2384  Exposure of the image difference between the warp and template.
2385  """
2386 
2387  # If the PSF-Matched warp did not exist for this direct warp
2388  # None is holding its place to maintain order in Gen 3
2389  if warpRef is None:
2390  return None
2391  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2392  warpName = self.getTempExpDatasetName('psfMatched')
2393  if not isinstance(warpRef, DeferredDatasetHandle):
2394  if not warpRef.datasetExists(warpName):
2395  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2396  return None
2397  warp = warpRef.get(datasetType=warpName, immediate=True)
2398  # direct image scaler OK for PSF-matched Warp
2399  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2400  mi = warp.getMaskedImage()
2401  if self.config.doScaleWarpVariance:
2402  try:
2403  self.scaleWarpVariance.run(mi)
2404  except Exception as exc:
2405  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2406  mi -= templateCoadd.getMaskedImage()
2407  return warp
2408 
2409  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2410  """Return a path to which to write debugging output.
2411 
2412  Creates a hyphen-delimited string of dataId values for simple filenames.
2413 
2414  Parameters
2415  ----------
2416  prefix : `str`
2417  Prefix for filename.
2418  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2419  Butler dataRef to make the path from.
2420  coaddLevel : `bool`, optional.
2421  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2422  'filter', but no 'visit').
2423 
2424  Returns
2425  -------
2426  result : `str`
2427  Path for debugging output.
2428  """
2429  if coaddLevel:
2430  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2431  else:
2432  keys = warpRef.dataId.keys()
2433  keyList = sorted(keys, reverse=True)
2434  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2435  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2436  return os.path.join(directory, filename)
2437 
2438 
2439 def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2440  """Match the order of one list to another, padding if necessary
2441 
2442  Parameters
2443  ----------
2444  inputList : list
2445  List to be reordered and padded. Elements can be any type.
2446  inputKeys : iterable
2447  Iterable of values to be compared with outputKeys.
2448  Length must match `inputList`
2449  outputKeys : iterable
2450  Iterable of values to be compared with inputKeys.
2451  padWith :
2452  Any value to be inserted where inputKey not in outputKeys
2453 
2454  Returns
2455  -------
2456  list
2457  Copy of inputList reordered per outputKeys and padded with `padWith`
2458  so that the length matches length of outputKeys.
2459  """
2460  outputList = []
2461  for d in outputKeys:
2462  if d in inputKeys:
2463  outputList.append(inputList[inputKeys.index(d)])
2464  else:
2465  outputList.append(padWith)
2466  return outputList
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.filterArtifacts
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
Definition: assembleCoadd.py:2316
lsst::afw::image
lsst.pipe.tasks.assembleCoadd.getTempExpRefList
def getTempExpRefList(self, patchRef, calExpRefList)
Definition: assembleCoadd.py:582
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig
Definition: assembleCoadd.py:1734
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig
Definition: assembleCoadd.py:1225
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.makeSupplementaryData
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
Definition: assembleCoadd.py:2061
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Definition: assembleCoadd.py:1428
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
Definition: assembleCoadd.py:2101
lsst.pipe.tasks.assembleCoadd.assembleSubregion
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Definition: assembleCoadd.py:856
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.validate
def validate(self)
Definition: assembleCoadd.py:1868
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.__init__
def __init__(self, *args, **kwargs)
Definition: assembleCoadd.py:1419
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask
Definition: assembleCoadd.py:1307
lsst.pipe.tasks.assembleCoadd.processResults
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
Definition: assembleCoadd.py:519
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.clipDetection
clipDetection
Definition: assembleCoadd.py:1236
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.findArtifacts
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Definition: assembleCoadd.py:2164
lsst.pipe.tasks.coaddBase.makeSkyInfo
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.statistic
statistic
Definition: assembleCoadd.py:1836
lsst.pipe.tasks.assembleCoadd.AssembleCoaddConnections
Definition: assembleCoadd.py:56
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._noTemplateMessage
def _noTemplateMessage(self, warpType)
Definition: assembleCoadd.py:2084
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.detectTemplate
detectTemplate
Definition: assembleCoadd.py:1744
lsst.pipe.tasks.assembleCoadd.reorderAndPadList
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: assembleCoadd.py:2439
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.warpType
warpType
Definition: assembleCoadd.py:1874
lsst.pipe.tasks.assembleCoadd.removeMaskPlanes
def removeMaskPlanes(self, maskedImage)
Definition: assembleCoadd.py:933
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.detectClip
def detectClip(self, exp, tempExpRefList)
Definition: assembleCoadd.py:1520
lsst.pipe.tasks.assembleCoadd.assembleMetadata
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Definition: assembleCoadd.py:795
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.sigmaClip
sigmaClip
Definition: assembleCoadd.py:1287
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.assembleMeanClipCoadd
assembleMeanClipCoadd
Definition: assembleCoadd.py:1232
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask
Definition: assembleCoadd.py:1881
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConnections.__init__
def __init__(self, *config=None)
Definition: assembleCoadd.py:1726
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.prefilterArtifacts
def prefilterArtifacts(self, spanSetList, exp)
Definition: assembleCoadd.py:2284
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._readAndComputeWarpDiff
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Definition: assembleCoadd.py:2369
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:714
lsst.pipe.tasks.assembleCoadd.prepareStats
def prepareStats(self, mask=None)
Definition: assembleCoadd.py:681
lsstDebug::Info
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.detect
detect
Definition: assembleCoadd.py:1740
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.doUsePsfMatchedPolygons
doUsePsfMatchedPolygons
Definition: assembleCoadd.py:1837
lsst.pipe.tasks.assembleCoadd.makeSupplementaryData
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
Definition: assembleCoadd.py:539
lsst::afw::table
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConnections
Definition: assembleCoadd.py:1707
lsst::utils
lsst.pipe.tasks.assembleCoadd.setInexactPsf
def setInexactPsf(self, mask)
Definition: assembleCoadd.py:1096
lsst::pex::config
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.assembleMeanCoadd
assembleMeanCoadd
Definition: assembleCoadd.py:1228
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.buildDifferenceImage
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Definition: assembleCoadd.py:1484
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.doSigmaClip
doSigmaClip
Definition: assembleCoadd.py:1299
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.detectClipBig
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Definition: assembleCoadd.py:1646
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.applyAltEdgeMask
def applyAltEdgeMask(self, mask, altMaskList)
Definition: assembleCoadd.py:2145
lsst::afw::detection
lsst.pipe.tasks.coaddHelpers.getGroupDataRef
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
lsst::geom
lsst.pipe.tasks.assembleCoadd.countMaskFromFootprint
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Definition: assembleCoadd.py:1192
lsst.pipe.tasks.assembleCoadd.readBrightObjectMasks
def readBrightObjectMasks(self, dataRef)
Definition: assembleCoadd.py:1032
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.setDefaults
def setDefaults(self)
Definition: assembleCoadd.py:1269
lsst.pipe.tasks.assembleCoadd.AssembleCoaddDataIdContainer
Definition: assembleCoadd.py:1164
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.validate
def validate(self)
Definition: assembleCoadd.py:1295
lsst.pipe.tasks.assembleCoadd.prepareInputs
def prepareInputs(self, refList)
Definition: assembleCoadd.py:606
lsst::afw::math
lsst.pipe.tasks.coaddBase.makeCoaddSuffix
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:336
lsst.pipe.tasks.assembleCoadd.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:558
Point< int, 2 >
lsst.pipe.tasks.coaddBase.CoaddBaseTask
Base class for coaddition.
Definition: coaddBase.py:131
lsst.pipe.tasks.assembleCoadd.setBrightObjectMasks
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
Definition: assembleCoadd.py:1054
lsst::geom::Box2I
lsst::pex::exceptions
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._dataRef2DebugPath
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Definition: assembleCoadd.py:2409
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.clipIter
clipIter
Definition: assembleCoadd.py:1288
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.__init__
def __init__(self, *args, **kwargs)
Definition: assembleCoadd.py:2005
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.statistic
statistic
Definition: assembleCoadd.py:1289
lsst::geom::Box2D
lsst.pipe::base
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.assembleStaticSkyModel
assembleStaticSkyModel
Definition: assembleCoadd.py:1735
lsst::log
lsst.pipe.tasks.assembleCoadd.applyAltMaskPlanes
def applyAltMaskPlanes(self, mask, altMaskSpans)
Definition: assembleCoadd.py:978
lsst::meas::algorithms
lsst.pipe.tasks.coaddHelpers.groupPatchExposures
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:59
Extent< int, 2 >
lsst::coadd::utils
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.setDefaults
def setDefaults(self)
Definition: assembleCoadd.py:1834
lsst.pipe.tasks.assembleCoadd.AssembleCoaddDataIdContainer.makeDataRefList
def makeDataRefList(self, namespace)
Definition: assembleCoadd.py:1168
lsst.pipe.tasks.assembleCoadd.shrinkValidPolygons
def shrinkValidPolygons(self, coaddInputs)
Definition: assembleCoadd.py:1010
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:2018
lsst::afw::geom