lsst.pipe.tasks  15.0-16-g6f0eb036+5
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy
24 import lsst.pex.config as pexConfig
25 import lsst.pex.exceptions as pexExceptions
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.math as afwMath
29 import lsst.afw.table as afwTable
30 import lsst.afw.detection as afwDet
31 import lsst.coadd.utils as coaddUtils
32 import lsst.pipe.base as pipeBase
33 import lsst.meas.algorithms as measAlg
34 import lsst.log as log
35 import lsstDebug
36 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
37 from .interpImage import InterpImageTask
38 from .scaleZeroPoint import ScaleZeroPointTask
39 from .coaddHelpers import groupPatchExposures, getGroupDataRef
40 from .scaleVariance import ScaleVarianceTask
41 from lsst.meas.algorithms import SourceDetectionTask
42 
43 __all__ = ["AssembleCoaddTask", "AssembleCoaddConfig", "SafeClipAssembleCoaddTask",
44  "SafeClipAssembleCoaddConfig", "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
45 
46 
47 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
48  """Configuration parameters for the `AssembleCoaddTask`.
49 
50  Notes
51  -----
52  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
53  only set the bitplane config.brightObjectMaskName. To make this useful you
54  *must* also configure the flags.pixel algorithm, for example by adding
55 
56  .. code-block:: none
57 
58  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
59  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
60 
61  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
62  """
63  warpType = pexConfig.Field(
64  doc="Warp name: one of 'direct' or 'psfMatched'",
65  dtype=str,
66  default="direct",
67  )
68  subregionSize = pexConfig.ListField(
69  dtype=int,
70  doc="Width, height of stack subregion size; "
71  "make small enough that a full stack of images will fit into memory at once.",
72  length=2,
73  default=(2000, 2000),
74  )
75  statistic = pexConfig.Field(
76  dtype=str,
77  doc="Main stacking statistic for aggregating over the epochs.",
78  default="MEANCLIP",
79  )
80  doSigmaClip = pexConfig.Field(
81  dtype=bool,
82  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
83  default=False,
84  )
85  sigmaClip = pexConfig.Field(
86  dtype=float,
87  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
88  default=3.0,
89  )
90  clipIter = pexConfig.Field(
91  dtype=int,
92  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
93  default=2,
94  )
95  calcErrorFromInputVariance = pexConfig.Field(
96  dtype=bool,
97  doc="Calculate coadd variance from input variance by stacking statistic."
98  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
99  default=True,
100  )
101  scaleZeroPoint = pexConfig.ConfigurableField(
102  target=ScaleZeroPointTask,
103  doc="Task to adjust the photometric zero point of the coadd temp exposures",
104  )
105  doInterp = pexConfig.Field(
106  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
107  dtype=bool,
108  default=True,
109  )
110  interpImage = pexConfig.ConfigurableField(
111  target=InterpImageTask,
112  doc="Task to interpolate (and extrapolate) over NaN pixels",
113  )
114  doWrite = pexConfig.Field(
115  doc="Persist coadd?",
116  dtype=bool,
117  default=True,
118  )
119  doNImage = pexConfig.Field(
120  doc="Create image of number of contributing exposures for each pixel",
121  dtype=bool,
122  default=False,
123  )
124  doUsePsfMatchedPolygons = pexConfig.Field(
125  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
126  dtype=bool,
127  default=False,
128  )
129  maskPropagationThresholds = pexConfig.DictField(
130  keytype=str,
131  itemtype=float,
132  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
133  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
134  "would have contributed exceeds this value."),
135  default={"SAT": 0.1},
136  )
137  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
138  doc="Mask planes to remove before coadding")
139  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
140  doc="Set mask and flag bits for bright objects?")
141  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
142  doc="Name of mask bit used for bright objects")
143  coaddPsf = pexConfig.ConfigField(
144  doc="Configuration for CoaddPsf",
145  dtype=measAlg.CoaddPsfConfig,
146  )
147  doAttachTransmissionCurve = pexConfig.Field(
148  dtype=bool, default=False, optional=False,
149  doc=("Attach a piecewise TransmissionCurve for the coadd? "
150  "(requires all input Exposures to have TransmissionCurves).")
151  )
152 
153  def setDefaults(self):
154  CoaddBaseTask.ConfigClass.setDefaults(self)
155  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
156 
157  def validate(self):
158  CoaddBaseTask.ConfigClass.validate(self)
159  if self.doPsfMatch:
160  # Backwards compatibility.
161  # Configs do not have loggers
162  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
163  self.warpType = 'psfMatched'
164  if self.doSigmaClip and self.statistic != "MEANCLIP":
165  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
166  self.statistic = "MEANCLIP"
167  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
168  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
169  "compute and set a non-zero coadd variance estimate." % (self.statistic))
170 
171  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
172  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
173  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
174  if str(k) not in unstackableStats]
175  raise ValueError("statistic %s is not allowed. Please choose one of %s."
176  % (self.statistic, stackableStats))
177 
178 
180  """Assemble a coadded image from a set of warps (coadded temporary exposures).
181 
182  We want to assemble a coadded image from a set of Warps (also called
183  coadded temporary exposures or ``coaddTempExps``).
184  Each input Warp covers a patch on the sky and corresponds to a single
185  run/visit/exposure of the covered patch. We provide the task with a list
186  of Warps (``selectDataList``) from which it selects Warps that cover the
187  specified patch (pointed at by ``dataRef``).
188  Each Warp that goes into a coadd will typically have an independent
189  photometric zero-point. Therefore, we must scale each Warp to set it to
190  a common photometric zeropoint. WarpType may be one of 'direct' or
191  'psfMatched', and the boolean configs `config.makeDirect` and
192  `config.makePsfMatched` set which of the warp types will be coadded.
193  The coadd is computed as a mean with optional outlier rejection.
194  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
195  Finally, Warps can have bad 'NaN' pixels which received no input from the
196  source calExps. We interpolate over these bad (NaN) pixels.
197 
198  `AssembleCoaddTask` uses several sub-tasks. These are
199 
200  - `ScaleZeroPointTask`
201  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
202  - `InterpImageTask`
203  - interpolate across bad pixels (NaN) in the final coadd
204 
205  You can retarget these subtasks if you wish.
206 
207  Notes
208  -----
209  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
210  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
211  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
212  no debug variables of its own. Some of the subtasks may support debug
213  variables. See the documentation for the subtasks for further information.
214 
215  Examples
216  --------
217  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
218  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
219  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
220  inputs: a data reference to the tract patch and filter to be coadded, and
221  a list of Warps to attempt to coadd. These are specified using ``--id`` and
222  ``--selectId``, respectively:
223 
224  .. code-block:: none
225 
226  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
227  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
228 
229  Only the Warps that cover the specified tract and patch will be coadded.
230  A list of the available optional arguments can be obtained by calling
231  ``assembleCoadd.py`` with the ``--help`` command line argument:
232 
233  .. code-block:: none
234 
235  assembleCoadd.py --help
236 
237  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
238  multi-band processing, we will generate the HSC-I & -R band coadds from
239  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
240  assuming that the lsst stack has been already set up, we must set up the
241  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
242  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
243  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
244  coadds, we must first
245 
246  - processCcd
247  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
248  - makeSkyMap
249  - create a skymap that covers the area of the sky present in the raw exposures
250  - makeCoaddTempExp
251  - warp the individual calibrated exposures to the tangent plane of the coadd
252 
253  We can perform all of these steps by running
254 
255  .. code-block:: none
256 
257  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
258 
259  This will produce warped exposures for each visit. To coadd the warped
260  data, we call assembleCoadd.py as follows:
261 
262  .. code-block:: none
263 
264  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
265  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
266  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
267  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
268  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
269  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
270  --selectId visit=903988 ccd=24
271 
272  that will process the HSC-I band data. The results are written in
273  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
274 
275  You may also choose to run:
276 
277  .. code-block:: none
278 
279  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
280  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
281  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
282  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
283  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
284  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
285  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
286  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
287 
288  to generate the coadd for the HSC-R band if you are interested in
289  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
290  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
291  rather than `AssembleCoaddTask` to make the coadd.
292  """
293  ConfigClass = AssembleCoaddConfig
294  _DefaultName = "assembleCoadd"
295 
296  def __init__(self, *args, **kwargs):
297  CoaddBaseTask.__init__(self, *args, **kwargs)
298  self.makeSubtask("interpImage")
299  self.makeSubtask("scaleZeroPoint")
300 
301  if self.config.doMaskBrightObjects:
302  mask = afwImage.Mask()
303  try:
304  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
305  except pexExceptions.LsstCppException:
306  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
307  mask.getMaskPlaneDict().keys())
308  del mask
309 
310  self.warpType = self.config.warpType
311 
312  @pipeBase.timeMethod
313  def run(self, dataRef, selectDataList=[]):
314  """Assemble a coadd from a set of Warps.
315 
316  Coadd a set of Warps. Compute weights to be applied to each Warp and
317  find scalings to match the photometric zeropoint to a reference Warp.
318  Assemble the Warps using `assemble`. Interpolate over NaNs and
319  optionally write the coadd to disk. Return the coadded exposure.
320 
321  Parameters
322  ----------
323  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
324  Data reference defining the patch for coaddition and the
325  reference Warp (if ``config.autoReference=False``).
326  Used to access the following data products:
327  - ``self.config.coaddName + "Coadd_skyMap"``
328  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
329  - ``self.config.coaddName + "Coadd"``
330  selectDataList : `list`
331  List of data references to Warps. Data to be coadded will be
332  selected from this list based on overlap with the patch defined
333  by dataRef.
334 
335  Returns
336  -------
337  retStruct : `lsst.pipe.base.Struct`
338  Result struct with components:
339 
340  - ``coaddExposure``: coadded exposure (``Exposure``).
341  - ``nImage``: exposure count image (``Image``).
342  """
343  skyInfo = self.getSkyInfo(dataRef)
344  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
345  if len(calExpRefList) == 0:
346  self.log.warn("No exposures to coadd")
347  return
348  self.log.info("Coadding %d exposures", len(calExpRefList))
349 
350  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
351  inputData = self.prepareInputs(tempExpRefList)
352  self.log.info("Found %d %s", len(inputData.tempExpRefList),
353  self.getTempExpDatasetName(self.warpType))
354  if len(inputData.tempExpRefList) == 0:
355  self.log.warn("No coadd temporary exposures found")
356  return
357 
358  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
359 
360  retStruct = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
361  inputData.weightList, supplementaryData=supplementaryData)
362 
363  if self.config.doInterp:
364  self.interpImage.run(retStruct.coaddExposure.getMaskedImage(), planeName="NO_DATA")
365  # The variance must be positive; work around for DM-3201.
366  varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
367  with numpy.errstate(invalid="ignore"):
368  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
369 
370  if self.config.doMaskBrightObjects:
371  brightObjectMasks = self.readBrightObjectMasks(dataRef)
372  self.setBrightObjectMasks(retStruct.coaddExposure, dataRef.dataId, brightObjectMasks)
373 
374  if self.config.doWrite:
375  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
376  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
377  if self.config.doNImage and retStruct.nImage is not None:
378  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
379 
380  return retStruct
381 
382  def makeSupplementaryData(self, dataRef, selectDataList):
383  """Make additional inputs to assemble() specific to subclasses.
384 
385  Available to be implemented by subclasses only if they need the
386  coadd dataRef for performing preliminary processing before
387  assembling the coadd.
388 
389  Parameters
390  ----------
391  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
392  Butler dataRef for supplementary data.
393  selectDataList : `list`
394  List of data references to Warps.
395  """
396  pass
397 
398  def getTempExpRefList(self, patchRef, calExpRefList):
399  """Generate list data references corresponding to warped exposures
400  that lie within the patch to be coadded.
401 
402  Parameters
403  ----------
404  patchRef : `dataRef`
405  Data reference for patch.
406  calExpRefList : `list`
407  List of data references for input calexps.
408 
409  Returns
410  -------
411  tempExpRefList : `list`
412  List of Warp/CoaddTempExp data references.
413  """
414  butler = patchRef.getButler()
415  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
416  self.getTempExpDatasetName(self.warpType))
417  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
418  g, groupData.keys) for
419  g in groupData.groups.keys()]
420  return tempExpRefList
421 
422  def prepareInputs(self, refList):
423  """Prepare the input warps for coaddition by measuring the weight for
424  each warp and the scaling for the photometric zero point.
425 
426  Each Warp has its own photometric zeropoint and background variance.
427  Before coadding these Warps together, compute a scale factor to
428  normalize the photometric zeropoint and compute the weight for each Warp.
429 
430  Parameters
431  ----------
432  refList : `list`
433  List of data references to tempExp
434 
435  Returns
436  -------
437  result : `lsst.pipe.base.Struct`
438  Result struct with components:
439 
440  - ``tempExprefList``: `list` of data references to tempExp.
441  - ``weightList``: `list` of weightings.
442  - ``imageScalerList``: `list` of image scalers.
443  """
444  statsCtrl = afwMath.StatisticsControl()
445  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
446  statsCtrl.setNumIter(self.config.clipIter)
447  statsCtrl.setAndMask(self.getBadPixelMask())
448  statsCtrl.setNanSafe(True)
449  # compute tempExpRefList: a list of tempExpRef that actually exist
450  # and weightList: a list of the weight of the associated coadd tempExp
451  # and imageScalerList: a list of scale factors for the associated coadd tempExp
452  tempExpRefList = []
453  weightList = []
454  imageScalerList = []
455  tempExpName = self.getTempExpDatasetName(self.warpType)
456  for tempExpRef in refList:
457  if not tempExpRef.datasetExists(tempExpName):
458  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
459  continue
460 
461  tempExp = tempExpRef.get(tempExpName, immediate=True)
462  maskedImage = tempExp.getMaskedImage()
463  imageScaler = self.scaleZeroPoint.computeImageScaler(
464  exposure=tempExp,
465  dataRef=tempExpRef,
466  )
467  try:
468  imageScaler.scaleMaskedImage(maskedImage)
469  except Exception as e:
470  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
471  continue
472  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
473  afwMath.MEANCLIP, statsCtrl)
474  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
475  weight = 1.0 / float(meanVar)
476  if not numpy.isfinite(weight):
477  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
478  continue
479  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
480 
481  del maskedImage
482  del tempExp
483 
484  tempExpRefList.append(tempExpRef)
485  weightList.append(weight)
486  imageScalerList.append(imageScaler)
487 
488  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
489  imageScalerList=imageScalerList)
490 
491  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
492  altMaskList=None, mask=None, supplementaryData=None):
493  """Assemble a coadd from input warps
494 
495  Assemble the coadd using the provided list of coaddTempExps. Since
496  the full coadd covers a patch (a large area), the assembly is
497  performed over small areas on the image at a time in order to
498  conserve memory usage. Iterate over subregions within the outer
499  bbox of the patch using `assembleSubregion` to stack the corresponding
500  subregions from the coaddTempExps with the statistic specified.
501  Set the edge bits the coadd mask based on the weight map.
502 
503  Parameters
504  ----------
505  skyInfo : `lsst.pipe.base.Struct`
506  Struct with geometric information about the patch.
507  tempExpRefList : `list`
508  List of data references to Warps (previously called CoaddTempExps).
509  imageScalerList : `list`
510  List of image scalers.
511  weightList : `list`
512  List of weights
513  altMaskList : `list`, optional
514  List of alternate masks to use rather than those stored with
515  tempExp.
516  mask : `lsst.afw.image.Mask`, optional
517  Mask to ignore when coadding
518  supplementaryData : lsst.pipe.base.Struct, optional
519  Struct with additional data products needed to assemble coadd.
520  Only used by subclasses that implement `makeSupplementaryData`
521  and override `assemble`.
522 
523  Returns
524  -------
525  result : `lsst.pipe.base.Struct`
526  Result struct with components:
527 
528  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
529  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
530  """
531  tempExpName = self.getTempExpDatasetName(self.warpType)
532  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
533  if mask is None:
534  mask = self.getBadPixelMask()
535 
536  statsCtrl = afwMath.StatisticsControl()
537  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
538  statsCtrl.setNumIter(self.config.clipIter)
539  statsCtrl.setAndMask(mask)
540  statsCtrl.setNanSafe(True)
541  statsCtrl.setWeighted(True)
542  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
543  for plane, threshold in self.config.maskPropagationThresholds.items():
544  bit = afwImage.Mask.getMaskPlane(plane)
545  statsCtrl.setMaskPropagationThreshold(bit, threshold)
546 
547  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
548 
549  if altMaskList is None:
550  altMaskList = [None]*len(tempExpRefList)
551 
552  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
553  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
554  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
555  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
556  coaddMaskedImage = coaddExposure.getMaskedImage()
557  subregionSizeArr = self.config.subregionSize
558  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
559  # if nImage is requested, create a zero one which can be passed to assembleSubregion
560  if self.config.doNImage:
561  nImage = afwImage.ImageU(skyInfo.bbox)
562  else:
563  nImage = None
564  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
565  try:
566  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
567  weightList, altMaskList, statsFlags, statsCtrl,
568  nImage=nImage)
569  except Exception as e:
570  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
571 
572  self.setInexactPsf(coaddMaskedImage.getMask())
573  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
574  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
575  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
576  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
577 
578  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
579  """Set the metadata for the coadd.
580 
581  This basic implementation sets the filter from the first input.
582 
583  Parameters
584  ----------
585  coaddExposure : `lsst.afw.image.Exposure`
586  The target exposure for the coadd.
587  tempExpRefList : `list`
588  List of data references to tempExp.
589  weightList : `list`
590  List of weights.
591  """
592  assert len(tempExpRefList) == len(weightList), "Length mismatch"
593  tempExpName = self.getTempExpDatasetName(self.warpType)
594  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
595  # (and we need more than just the PropertySet that contains the header), which is not possible
596  # with the current butler (see #2777).
597  tempExpList = [tempExpRef.get(tempExpName + "_sub",
598  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
599  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
600  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
601 
602  coaddExposure.setFilter(tempExpList[0].getFilter())
603  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
604  coaddInputs.ccds.reserve(numCcds)
605  coaddInputs.visits.reserve(len(tempExpList))
606 
607  for tempExp, weight in zip(tempExpList, weightList):
608  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
609 
610  if self.config.doUsePsfMatchedPolygons:
611  self.shrinkValidPolygons(coaddInputs)
612 
613  coaddInputs.visits.sort()
614  if self.warpType == "psfMatched":
615  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
616  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
617  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
618  # having the maximum width (sufficient because square)
619  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
620  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
621  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
622  else:
623  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
624  self.config.coaddPsf.makeControl())
625  coaddExposure.setPsf(psf)
626  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
627  coaddExposure.getWcs())
628  coaddExposure.getInfo().setApCorrMap(apCorrMap)
629  if self.config.doAttachTransmissionCurve:
630  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
631  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
632 
633  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
634  altMaskList, statsFlags, statsCtrl, nImage=None):
635  """Assemble the coadd for a sub-region.
636 
637  For each coaddTempExp, check for (and swap in) an alternative mask
638  if one is passed. Remove mask planes listed in
639  `config.removeMaskPlanes`. Finally, stack the actual exposures using
640  `lsst.afw.math.statisticsStack` with the statistic specified by
641  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
642  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
643  an N-sigma clipped mean where N and iterations are specified by
644  statsCtrl. Assign the stacked subregion back to the coadd.
645 
646  Parameters
647  ----------
648  coaddExposure : `lsst.afw.image.Exposure`
649  The target exposure for the coadd.
650  bbox : `lsst.afw.geom.Box`
651  Sub-region to coadd.
652  tempExpRefList : `list`
653  List of data reference to tempExp.
654  imageScalerList : `list`
655  List of image scalers.
656  weightList : `list`
657  List of weights.
658  altMaskList : `list`
659  List of alternate masks to use rather than those stored with
660  tempExp, or None. Each element is dict with keys = mask plane
661  name to which to add the spans.
662  statsFlags : `lsst.afw.math.Property`
663  Property object for statistic for coadd.
664  statsCtrl : `lsst.afw.math.StatisticsControl`
665  Statistics control object for coadd.
666  nImage : `lsst.afw.image.ImageU`, optional
667  Keeps track of exposure count for each pixel.
668  """
669  self.log.debug("Computing coadd over %s", bbox)
670  tempExpName = self.getTempExpDatasetName(self.warpType)
671  coaddExposure.mask.addMaskPlane("REJECTED")
672  coaddExposure.mask.addMaskPlane("CLIPPED")
673  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
674  # If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
675  # or CLIPPED, set it to REJECTED on the coadd.
676  # If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
677  # if a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
678  edge = afwImage.Mask.getPlaneBitMask("EDGE")
679  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
680  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
681  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
682  maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask("REJECTED")),
683  (edge, coaddExposure.mask.getPlaneBitMask("SENSOR_EDGE")),
684  (clipped, clipped)]
685  maskedImageList = []
686  if nImage is not None:
687  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
688  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
689  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
690  maskedImage = exposure.getMaskedImage()
691  mask = maskedImage.getMask()
692  if altMask is not None:
693  self.applyAltMaskPlanes(mask, altMask)
694  imageScaler.scaleMaskedImage(maskedImage)
695 
696  # Add 1 for each pixel which is not excluded by the exclude mask.
697  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
698  if nImage is not None:
699  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
700  if self.config.removeMaskPlanes:
701  mask = maskedImage.getMask()
702  for maskPlane in self.config.removeMaskPlanes:
703  try:
704  mask &= ~mask.getPlaneBitMask(maskPlane)
705  except Exception as e:
706  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
707 
708  maskedImageList.append(maskedImage)
709 
710  with self.timer("stack"):
711  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
712  clipped, # also set output to CLIPPED if sigma-clipped
713  maskMap)
714  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
715  if nImage is not None:
716  nImage.assign(subNImage, bbox)
717 
718  def applyAltMaskPlanes(self, mask, altMaskSpans):
719  """Apply in place alt mask formatted as SpanSets to a mask.
720 
721  Parameters
722  ----------
723  mask : `lsst.afw.image.Mask`
724  Original mask.
725  altMaskSpans : `dict`
726  SpanSet lists to apply. Each element contains the new mask
727  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
728  and list of SpanSets to apply to the mask.
729 
730  Returns
731  -------
732  mask : `lsst.afw.image.Mask`
733  Updated mask.
734  """
735  if self.config.doUsePsfMatchedPolygons:
736  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
737  # Clear away any other masks outside the validPolygons. These pixels are no longer
738  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
739  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
740  # This mask-clearing step must only occur *before* applying the new masks below
741  for spanSet in altMaskSpans['NO_DATA']:
742  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
743 
744  for plane, spanSetList in altMaskSpans.items():
745  maskClipValue = mask.addMaskPlane(plane)
746  for spanSet in spanSetList:
747  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
748  return mask
749 
750  def shrinkValidPolygons(self, coaddInputs):
751  """Shrink coaddInputs' ccds' ValidPolygons in place.
752 
753  Either modify each ccd's validPolygon in place, or if CoaddInputs
754  does not have a validPolygon, create one from its bbox.
755 
756  Parameters
757  ----------
758  coaddInputs : `lsst.afw.image.coaddInputs`
759  Original mask.
760 
761  """
762  for ccd in coaddInputs.ccds:
763  polyOrig = ccd.getValidPolygon()
764  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
765  validPolyBBox.grow(-self.config.matchingKernelSize//2)
766  if polyOrig:
767  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
768  else:
769  validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
770  ccd.setValidPolygon(validPolygon)
771 
772  def readBrightObjectMasks(self, dataRef):
773  """Retrieve the bright object masks.
774 
775  Returns None on failure.
776 
777  Parameters
778  ----------
779  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
780  A Butler dataRef.
781 
782  Returns
783  -------
784  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
785  Bright object mask from the Butler object, or None if it cannot
786  be retrieved.
787  """
788  try:
789  return dataRef.get("brightObjectMask", immediate=True)
790  except Exception as e:
791  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
792  return None
793 
794  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
795  """Set the bright object masks.
796 
797  Parameters
798  ----------
799  exposure : `lsst.afw.image.Exposure`
800  Exposure under consideration.
801  dataId : `lsst.daf.persistence.dataId`
802  Data identifier dict for patch.
803  brightObjectMasks : `lsst.afw.table`
804  Table of bright objects to mask.
805  """
806  #
807  # Check the metadata specifying the tract/patch/filter
808  #
809  if brightObjectMasks is None:
810  self.log.warn("Unable to apply bright object mask: none supplied")
811  return
812  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
813  md = brightObjectMasks.table.getMetadata()
814  for k in dataId:
815  if not md.exists(k):
816  self.log.warn("Expected to see %s in metadata", k)
817  else:
818  if md.get(k) != dataId[k]:
819  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
820 
821  mask = exposure.getMaskedImage().getMask()
822  wcs = exposure.getWcs()
823  plateScale = wcs.getPixelScale().asArcseconds()
824 
825  for rec in brightObjectMasks:
826  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
827  if rec["type"] == "box":
828  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
829  width = rec["width"].asArcseconds()/plateScale # convert to pixels
830  height = rec["height"].asArcseconds()/plateScale # convert to pixels
831 
832  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
833  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
834 
835  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
836  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
837  spans = afwGeom.SpanSet(bbox)
838  elif rec["type"] == "circle":
839  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
840  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
841  else:
842  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
843  continue
844  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
845 
846  def setInexactPsf(self, mask):
847  """Set INEXACT_PSF mask plane.
848 
849  If any of the input images isn't represented in the coadd (due to
850  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
851  these pixels.
852 
853  Parameters
854  ----------
855  mask : `lsst.afw.image.Mask`
856  Coadded exposure's mask, modified in-place.
857  """
858  mask.addMaskPlane("INEXACT_PSF")
859  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
860  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
861  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
862  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
863  array = mask.getArray()
864  selected = array & (sensorEdge | clipped | rejected) > 0
865  array[selected] |= inexactPsf
866 
867  @classmethod
868  def _makeArgumentParser(cls):
869  """Create an argument parser.
870  """
871  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
872  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
873  cls.ConfigClass().warpType + "Warp",
874  help="data ID, e.g. --id tract=12345 patch=1,2",
875  ContainerClass=AssembleCoaddDataIdContainer)
876  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
877  ContainerClass=SelectDataIdContainer)
878  return parser
879 
880 
881 def _subBBoxIter(bbox, subregionSize):
882  """Iterate over subregions of a bbox.
883 
884  Parameters
885  ----------
886  bbox : `lsst.afw.geom.Box2I`
887  Bounding box over which to iterate.
888  subregionSize: `lsst.afw.geom.Extent2I`
889  Size of sub-bboxes.
890 
891  Yields
892  ------
893  subBBox : `lsst.afw.geom.Box2I`
894  Next sub-bounding box of size subregionSize or smaller; each subBBox
895  is contained within bbox, so it may be smaller than subregionSize at
896  the edges of bbox, but it will never be empty.
897  """
898  if bbox.isEmpty():
899  raise RuntimeError("bbox %s is empty" % (bbox,))
900  if subregionSize[0] < 1 or subregionSize[1] < 1:
901  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
902 
903  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
904  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
905  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
906  subBBox.clip(bbox)
907  if subBBox.isEmpty():
908  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
909  (bbox, subregionSize, colShift, rowShift))
910  yield subBBox
911 
912 
913 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
914  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
915  """
916 
917  def makeDataRefList(self, namespace):
918  """Make self.refList from self.idList.
919 
920  Parameters
921  ----------
922  namespace
923  Results of parsing command-line (with ``butler`` and ``log`` elements).
924  """
925  datasetType = namespace.config.coaddName + "Coadd"
926  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
927 
928  for dataId in self.idList:
929  # tract and patch are required
930  for key in keysCoadd:
931  if key not in dataId:
932  raise RuntimeError("--id must include " + key)
933 
934  dataRef = namespace.butler.dataRef(
935  datasetType=datasetType,
936  dataId=dataId,
937  )
938  self.refList.append(dataRef)
939 
940 
941 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
942  """Function to count the number of pixels with a specific mask in a
943  footprint.
944 
945  Find the intersection of mask & footprint. Count all pixels in the mask
946  that are in the intersection that have bitmask set but do not have
947  ignoreMask set. Return the count.
948 
949  Parameters
950  ----------
951  mask : `lsst.afw.image.Mask`
952  Mask to define intersection region by.
953  footprint : `lsst.afw.detection.Footprint`
954  Footprint to define the intersection region by.
955  bitmask
956  Specific mask that we wish to count the number of occurances of.
957  ignoreMask
958  Pixels to not consider.
959 
960  Returns
961  -------
962  result : `int`
963  Count of number of pixels in footprint with specified mask.
964  """
965  bbox = footprint.getBBox()
966  bbox.clip(mask.getBBox(afwImage.PARENT))
967  fp = afwImage.Mask(bbox)
968  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
969  footprint.spans.setMask(fp, bitmask)
970  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
971  (subMask.getArray() & ignoreMask) == 0).sum()
972 
973 
975  """Configuration parameters for the SafeClipAssembleCoaddTask.
976  """
977  clipDetection = pexConfig.ConfigurableField(
978  target=SourceDetectionTask,
979  doc="Detect sources on difference between unclipped and clipped coadd")
980  minClipFootOverlap = pexConfig.Field(
981  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
982  dtype=float,
983  default=0.6
984  )
985  minClipFootOverlapSingle = pexConfig.Field(
986  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
987  "clipped when only one visit overlaps",
988  dtype=float,
989  default=0.5
990  )
991  minClipFootOverlapDouble = pexConfig.Field(
992  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
993  "clipped when two visits overlap",
994  dtype=float,
995  default=0.45
996  )
997  maxClipFootOverlapDouble = pexConfig.Field(
998  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
999  "considering two visits",
1000  dtype=float,
1001  default=0.15
1002  )
1003  minBigOverlap = pexConfig.Field(
1004  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1005  "when labeling clipped footprints",
1006  dtype=int,
1007  default=100
1008  )
1009 
1010  def setDefaults(self):
1011  """Set default values for clipDetection.
1012 
1013  Notes
1014  -----
1015  The numeric values for these configuration parameters were
1016  empirically determined, future work may further refine them.
1017  """
1018  AssembleCoaddConfig.setDefaults(self)
1019  self.clipDetection.doTempLocalBackground = False
1020  self.clipDetection.reEstimateBackground = False
1021  self.clipDetection.returnOriginalFootprints = False
1022  self.clipDetection.thresholdPolarity = "both"
1023  self.clipDetection.thresholdValue = 2
1024  self.clipDetection.nSigmaToGrow = 2
1025  self.clipDetection.minPixels = 4
1026  self.clipDetection.isotropicGrow = True
1027  self.clipDetection.thresholdType = "pixel_stdev"
1028  self.sigmaClip = 1.5
1029  self.clipIter = 3
1030  self.statistic = "MEAN"
1031 
1032  def validate(self):
1033  if self.doSigmaClip:
1034  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1035  "Ignoring doSigmaClip.")
1036  self.doSigmaClip = False
1037  if self.statistic != "MEAN":
1038  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1039  "(%s chosen). Please set statistic to MEAN."
1040  % (self.statistic))
1041  AssembleCoaddTask.ConfigClass.validate(self)
1042 
1043 
1045  """Assemble a coadded image from a set of coadded temporary exposures,
1046  being careful to clip & flag areas with potential artifacts.
1047 
1048  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1049  we clip outliers). The problem with doing this is that when computing the
1050  coadd PSF at a given location, individual visit PSFs from visits with
1051  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1052  In this task, we correct for this behavior by creating a new
1053  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1054  coaddTempExps and the final coadd where
1055 
1056  i. difference imaging suggests that there is an outlier and
1057  ii. this outlier appears on only one or two images.
1058 
1059  Such regions will not contribute to the final coadd. Furthermore, any
1060  routine to determine the coadd PSF can now be cognizant of clipped regions.
1061  Note that the algorithm implemented by this task is preliminary and works
1062  correctly for HSC data. Parameter modifications and or considerable
1063  redesigning of the algorithm is likley required for other surveys.
1064 
1065  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1066  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1067  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1068  if you wish.
1069 
1070  Notes
1071  -----
1072  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1073  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1074  see `baseDebug` for more about ``debug.py`` files.
1075  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1076  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1077  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1078  for further information.
1079 
1080  Examples
1081  --------
1082  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1083  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1084  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1085 
1086  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1087  and filter to be coadded (specified using
1088  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1089  along with a list of coaddTempExps to attempt to coadd (specified using
1090  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1091  Only the coaddTempExps that cover the specified tract and patch will be
1092  coadded. A list of the available optional arguments can be obtained by
1093  calling assembleCoadd.py with the --help command line argument:
1094 
1095  .. code-block:: none
1096 
1097  assembleCoadd.py --help
1098 
1099  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1100  context of multi-band processing, we will generate the HSC-I & -R band
1101  coadds from HSC engineering test data provided in the ci_hsc package.
1102  To begin, assuming that the lsst stack has been already set up, we must
1103  set up the obs_subaru and ci_hsc packages. This defines the environment
1104  variable $CI_HSC_DIR and points at the location of the package. The raw
1105  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1106  the coadds, we must first
1107 
1108  - ``processCcd``
1109  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1110  - ``makeSkyMap``
1111  create a skymap that covers the area of the sky present in the raw exposures
1112  - ``makeCoaddTempExp``
1113  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1114 
1115  We can perform all of these steps by running
1116 
1117  .. code-block:: none
1118 
1119  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1120 
1121  This will produce warped coaddTempExps for each visit. To coadd the
1122  warped data, we call ``assembleCoadd.py`` as follows:
1123 
1124  .. code-block:: none
1125 
1126  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1127  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1128  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1129  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1130  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1131  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1132  --selectId visit=903988 ccd=24
1133 
1134  This will process the HSC-I band data. The results are written in
1135  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1136 
1137  You may also choose to run:
1138 
1139  .. code-block:: none
1140 
1141  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1142  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1143  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1144  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1145  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1146  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1147  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1148  --selectId visit=903346 ccd=12
1149 
1150  to generate the coadd for the HSC-R band if you are interested in following
1151  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1152  """
1153  ConfigClass = SafeClipAssembleCoaddConfig
1154  _DefaultName = "safeClipAssembleCoadd"
1155 
1156  def __init__(self, *args, **kwargs):
1157  AssembleCoaddTask.__init__(self, *args, **kwargs)
1158  schema = afwTable.SourceTable.makeMinimalSchema()
1159  self.makeSubtask("clipDetection", schema=schema)
1160 
1161  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1162  """Assemble the coadd for a region.
1163 
1164  Compute the difference of coadds created with and without outlier
1165  rejection to identify coadd pixels that have outlier values in some
1166  individual visits.
1167  Detect clipped regions on the difference image and mark these regions
1168  on the one or two individual coaddTempExps where they occur if there
1169  is significant overlap between the clipped region and a source. This
1170  leaves us with a set of footprints from the difference image that have
1171  been identified as having occured on just one or two individual visits.
1172  However, these footprints were generated from a difference image. It
1173  is conceivable for a large diffuse source to have become broken up
1174  into multiple footprints acrosss the coadd difference in this process.
1175  Determine the clipped region from all overlapping footprints from the
1176  detected sources in each visit - these are big footprints.
1177  Combine the small and big clipped footprints and mark them on a new
1178  bad mask plane.
1179  Generate the coadd using `AssembleCoaddTask.assemble` without outlier
1180  removal. Clipped footprints will no longer make it into the coadd
1181  because they are marked in the new bad mask plane.
1182 
1183  Parameters
1184  ----------
1185  skyInfo : `lsst.pipe.base.Struct`
1186  Patch geometry information, from getSkyInfo
1187  tempExpRefList : `list`
1188  List of data reference to tempExp
1189  imageScalerList : `list`
1190  List of image scalers
1191  weightList : `list`
1192  List of weights
1193 
1194  Returns
1195  -------
1196  result : `lsst.pipe.base.Struct`
1197  Result struct with components:
1198 
1199  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1200  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
1201 
1202  Notes
1203  -----
1204  args and kwargs are passed but ignored in order to match the call
1205  signature expected by the parent task.
1206  """
1207  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1208  mask = exp.getMaskedImage().getMask()
1209  mask.addMaskPlane("CLIPPED")
1210 
1211  result = self.detectClip(exp, tempExpRefList)
1212 
1213  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1214 
1215  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1216  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1217  # Append big footprints from individual Warps to result.clipSpans
1218  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1219  result.detectionFootprints, maskClipValue, maskDetValue,
1220  exp.getBBox())
1221  # Create mask of the current clipped footprints
1222  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1223  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1224 
1225  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1226  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1227  maskClip |= maskClipBig
1228 
1229  # Assemble coadd from base class, but ignoring CLIPPED pixels
1230  badMaskPlanes = self.config.badMaskPlanes[:]
1231  badMaskPlanes.append("CLIPPED")
1232  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1233  return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1234  result.clipSpans, mask=badPixelMask)
1235 
1236  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1237  """Return an exposure that contains the difference between unclipped
1238  and clipped coadds.
1239 
1240  Generate a difference image between clipped and unclipped coadds.
1241  Compute the difference image by subtracting an outlier-clipped coadd
1242  from an outlier-unclipped coadd. Return the difference image.
1243 
1244  Parameters
1245  ----------
1246  skyInfo : `lsst.pipe.base.Struct`
1247  Patch geometry information, from getSkyInfo
1248  tempExpRefList : `list`
1249  List of data reference to tempExp
1250  imageScalerList : `list`
1251  List of image scalers
1252  weightList : `list`
1253  List of weights
1254 
1255  Returns
1256  -------
1257  exp : `lsst.afw.image.Exposure`
1258  Difference image of unclipped and clipped coadd wrapped in an Exposure
1259  """
1260  # Clone and upcast self.config because current self.config is frozen
1261  config = AssembleCoaddConfig()
1262  # getattr necessary because subtasks do not survive Config.toDict()
1263  configIntersection = {k: getattr(self.config, k)
1264  for k, v in self.config.toDict().items() if (k in config.keys())}
1265  config.update(**configIntersection)
1266 
1267  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1268  config.statistic = 'MEAN'
1269  task = AssembleCoaddTask(config=config)
1270  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1271 
1272  config.statistic = 'MEANCLIP'
1273  task = AssembleCoaddTask(config=config)
1274  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1275 
1276  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1277  coaddDiff -= coaddClip.getMaskedImage()
1278  exp = afwImage.ExposureF(coaddDiff)
1279  exp.setPsf(coaddMean.getPsf())
1280  return exp
1281 
1282  def detectClip(self, exp, tempExpRefList):
1283  """Detect clipped regions on an exposure and set the mask on the
1284  individual tempExp masks.
1285 
1286  Detect footprints in the difference image after smoothing the
1287  difference image with a Gaussian kernal. Identify footprints that
1288  overlap with one or two input ``coaddTempExps`` by comparing the
1289  computed overlap fraction to thresholds set in the config. A different
1290  threshold is applied depending on the number of overlapping visits
1291  (restricted to one or two). If the overlap exceeds the thresholds,
1292  the footprint is considered "CLIPPED" and is marked as such on the
1293  coaddTempExp. Return a struct with the clipped footprints, the indices
1294  of the ``coaddTempExps`` that end up overlapping with the clipped
1295  footprints, and a list of new masks for the ``coaddTempExps``.
1296 
1297  Parameters
1298  ----------
1299  exp : `lsst.afw.image.Exposure`
1300  Exposure to run detection on.
1301  tempExpRefList : `list`
1302  List of data reference to tempExp.
1303 
1304  Returns
1305  -------
1306  result : `lsst.pipe.base.Struct`
1307  Result struct with components:
1308 
1309  - ``clipFootprints``: list of clipped footprints.
1310  - ``clipIndices``: indices for each ``clippedFootprint`` in
1311  ``tempExpRefList``.
1312  - ``clipSpans``: List of dictionaries containing spanSet lists
1313  to clip. Each element contains the new maskplane name
1314  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1315  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1316  compressed into footprints.
1317  """
1318  mask = exp.getMaskedImage().getMask()
1319  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1320  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1321  # Merge positive and negative together footprints together
1322  fpSet.positive.merge(fpSet.negative)
1323  footprints = fpSet.positive
1324  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1325  ignoreMask = self.getBadPixelMask()
1326 
1327  clipFootprints = []
1328  clipIndices = []
1329  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1330 
1331  # for use by detectClipBig
1332  visitDetectionFootprints = []
1333 
1334  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1335  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1336  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1337 
1338  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1339  for i, warpRef in enumerate(tempExpRefList):
1340  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1341  immediate=True).getMaskedImage().getMask()
1342  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1343  afwImage.PARENT, True)
1344  maskVisitDet &= maskDetValue
1345  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1346  visitDetectionFootprints.append(visitFootprints)
1347 
1348  for j, footprint in enumerate(footprints.getFootprints()):
1349  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1350  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1351 
1352  # build a list of clipped spans for each visit
1353  for j, footprint in enumerate(footprints.getFootprints()):
1354  nPixel = footprint.getArea()
1355  overlap = [] # hold the overlap with each visit
1356  indexList = [] # index of visit in global list
1357  for i in range(len(tempExpRefList)):
1358  ignore = ignoreArr[i, j]
1359  overlapDet = overlapDetArr[i, j]
1360  totPixel = nPixel - ignore
1361 
1362  # If we have more bad pixels than detection skip
1363  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1364  continue
1365  overlap.append(overlapDet/float(totPixel))
1366  indexList.append(i)
1367 
1368  overlap = numpy.array(overlap)
1369  if not len(overlap):
1370  continue
1371 
1372  keep = False # Should this footprint be marked as clipped?
1373  keepIndex = [] # Which tempExps does the clipped footprint belong to
1374 
1375  # If footprint only has one overlap use a lower threshold
1376  if len(overlap) == 1:
1377  if overlap[0] > self.config.minClipFootOverlapSingle:
1378  keep = True
1379  keepIndex = [0]
1380  else:
1381  # This is the general case where only visit should be clipped
1382  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1383  if len(clipIndex) == 1:
1384  keep = True
1385  keepIndex = [clipIndex[0]]
1386 
1387  # Test if there are clipped objects that overlap two different visits
1388  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1389  if len(clipIndex) == 2 and len(overlap) > 3:
1390  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1391  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1392  keep = True
1393  keepIndex = clipIndex
1394 
1395  if not keep:
1396  continue
1397 
1398  for index in keepIndex:
1399  globalIndex = indexList[index]
1400  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1401 
1402  clipIndices.append(numpy.array(indexList)[keepIndex])
1403  clipFootprints.append(footprint)
1404 
1405  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1406  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1407 
1408  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1409  maskClipValue, maskDetValue, coaddBBox):
1410  """Return individual warp footprints for large artifacts and append
1411  them to ``clipList`` in place.
1412 
1413  Identify big footprints composed of many sources in the coadd
1414  difference that may have originated in a large diffuse source in the
1415  coadd. We do this by indentifying all clipped footprints that overlap
1416  significantly with each source in all the coaddTempExps.
1417 
1418  Parameters
1419  ----------
1420  clipList : `list`
1421  List of alt mask SpanSets with clipping information. Modified.
1422  clipFootprints : `list`
1423  List of clipped footprints.
1424  clipIndices : `list`
1425  List of which entries in tempExpClipList each footprint belongs to.
1426  maskClipValue
1427  Mask value of clipped pixels.
1428  maskDetValue
1429  Mask value of detected pixels.
1430  coaddBBox : `lsst.afw.geom.Box`
1431  BBox of the coadd and warps.
1432 
1433  Returns
1434  -------
1435  bigFootprintsCoadd : `list`
1436  List of big footprints
1437  """
1438  bigFootprintsCoadd = []
1439  ignoreMask = self.getBadPixelMask()
1440  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1441  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1442  for footprint in visitFootprints.getFootprints():
1443  footprint.spans.setMask(maskVisitDet, maskDetValue)
1444 
1445  # build a mask of clipped footprints that are in this visit
1446  clippedFootprintsVisit = []
1447  for foot, clipIndex in zip(clipFootprints, clipIndices):
1448  if index not in clipIndex:
1449  continue
1450  clippedFootprintsVisit.append(foot)
1451  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1452  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1453 
1454  bigFootprintsVisit = []
1455  for foot in visitFootprints.getFootprints():
1456  if foot.getArea() < self.config.minBigOverlap:
1457  continue
1458  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1459  if nCount > self.config.minBigOverlap:
1460  bigFootprintsVisit.append(foot)
1461  bigFootprintsCoadd.append(foot)
1462 
1463  for footprint in bigFootprintsVisit:
1464  clippedSpans["CLIPPED"].append(footprint.spans)
1465 
1466  return bigFootprintsCoadd
1467 
1468 
1470  assembleStaticSkyModel = pexConfig.ConfigurableField(
1471  target=AssembleCoaddTask,
1472  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1473  " naive/first-iteration model of the static sky.",
1474  )
1475  detect = pexConfig.ConfigurableField(
1476  target=SourceDetectionTask,
1477  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1478  )
1479  detectTemplate = pexConfig.ConfigurableField(
1480  target=SourceDetectionTask,
1481  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1482  )
1483  maxNumEpochs = pexConfig.Field(
1484  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1485  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1486  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1487  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1488  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1489  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1490  "than transient and not masked.",
1491  dtype=int,
1492  default=2
1493  )
1494  maxFractionEpochsLow = pexConfig.RangeField(
1495  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1496  "Effective maxNumEpochs = "
1497  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1498  dtype=float,
1499  default=0.4,
1500  min=0., max=1.,
1501  )
1502  maxFractionEpochsHigh = pexConfig.RangeField(
1503  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1504  "Effective maxNumEpochs = "
1505  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1506  dtype=float,
1507  default=0.03,
1508  min=0., max=1.,
1509  )
1510  spatialThreshold = pexConfig.RangeField(
1511  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1512  "temporal criteria. If 0, clip all. If 1, clip none.",
1513  dtype=float,
1514  default=0.5,
1515  min=0., max=1.,
1516  inclusiveMin=True, inclusiveMax=True
1517  )
1518  doScaleWarpVariance = pexConfig.Field(
1519  doc="Rescale Warp variance plane using empirical noise?",
1520  dtype=bool,
1521  default=True,
1522  )
1523  scaleWarpVariance = pexConfig.ConfigurableField(
1524  target=ScaleVarianceTask,
1525  doc="Rescale variance on warps",
1526  )
1527  doPreserveContainedBySource = pexConfig.Field(
1528  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1529  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1530  dtype=bool,
1531  default=True,
1532  )
1533  doPrefilterArtifacts = pexConfig.Field(
1534  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1535  "because they will be excluded anyway. This prevents them from contributing "
1536  "to the outlier epoch count image and potentially being labeled as persistant."
1537  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1538  dtype=bool,
1539  default=True
1540  )
1541  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1542  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1543  dtype=str,
1544  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1545  )
1546  prefilterArtifactsRatio = pexConfig.Field(
1547  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1548  dtype=float,
1549  default=0.05
1550  )
1551 
1552  def setDefaults(self):
1553  AssembleCoaddConfig.setDefaults(self)
1554  self.statistic = 'MEAN'
1556 
1557  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1558  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1559  if "EDGE" in self.badMaskPlanes:
1560  self.badMaskPlanes.remove('EDGE')
1561  self.removeMaskPlanes.append('EDGE')
1562  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1563  self.assembleStaticSkyModel.warpType = 'psfMatched'
1564  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1565  self.assembleStaticSkyModel.sigmaClip = 2.5
1566  self.assembleStaticSkyModel.clipIter = 3
1567  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1568  self.assembleStaticSkyModel.doWrite = False
1569  self.detect.doTempLocalBackground = False
1570  self.detect.reEstimateBackground = False
1571  self.detect.returnOriginalFootprints = False
1572  self.detect.thresholdPolarity = "both"
1573  self.detect.thresholdValue = 5
1574  self.detect.nSigmaToGrow = 2
1575  self.detect.minPixels = 4
1576  self.detect.isotropicGrow = True
1577  self.detect.thresholdType = "pixel_stdev"
1578  self.detectTemplate.nSigmaToGrow = 2.0
1579  self.detectTemplate.doTempLocalBackground = False
1580  self.detectTemplate.reEstimateBackground = False
1581  self.detectTemplate.returnOriginalFootprints = False
1582 
1583 
1585  """Assemble a compareWarp coadded image from a set of warps
1586  by masking artifacts detected by comparing PSF-matched warps.
1587 
1588  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1589  we clip outliers). The problem with doing this is that when computing the
1590  coadd PSF at a given location, individual visit PSFs from visits with
1591  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1592  In this task, we correct for this behavior by creating a new badMaskPlane
1593  'CLIPPED' which marks pixels in the individual warps suspected to contain
1594  an artifact. We populate this plane on the input warps by comparing
1595  PSF-matched warps with a PSF-matched median coadd which serves as a
1596  model of the static sky. Any group of pixels that deviates from the
1597  PSF-matched template coadd by more than config.detect.threshold sigma,
1598  is an artifact candidate. The candidates are then filtered to remove
1599  variable sources and sources that are difficult to subtract such as
1600  bright stars. This filter is configured using the config parameters
1601  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1602  the maximum fraction of epochs that the deviation can appear in and still
1603  be considered an artifact. The spatialThreshold is the maximum fraction of
1604  pixels in the footprint of the deviation that appear in other epochs
1605  (where other epochs is defined by the temporalThreshold). If the deviant
1606  region meets this criteria of having a significant percentage of pixels
1607  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1608  set in the mask. These regions will not contribute to the final coadd.
1609  Furthermore, any routine to determine the coadd PSF can now be cognizant
1610  of clipped regions. Note that the algorithm implemented by this task is
1611  preliminary and works correctly for HSC data. Parameter modifications and
1612  or considerable redesigning of the algorithm is likley required for other
1613  surveys.
1614 
1615  ``CompareWarpAssembleCoaddTask`` sub-classes
1616  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1617  as a subtask to generate the TemplateCoadd (the model of the static sky).
1618 
1619  Notes
1620  -----
1621  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1622  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1623  ``baseDebug`` for more about ``debug.py`` files.
1624 
1625  This task supports the following debug variables:
1626 
1627  - ``saveCountIm``
1628  If True then save the Epoch Count Image as a fits file in the `figPath`
1629  - ``figPath``
1630  Path to save the debug fits images and figures
1631 
1632  For example, put something like:
1633 
1634  .. code-block:: python
1635 
1636  import lsstDebug
1637  def DebugInfo(name):
1638  di = lsstDebug.getInfo(name)
1639  if name == "lsst.pipe.tasks.assembleCoadd":
1640  di.saveCountIm = True
1641  di.figPath = "/desired/path/to/debugging/output/images"
1642  return di
1643  lsstDebug.Info = DebugInfo
1644 
1645  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1646  ``--debug`` flag. Some subtasks may have their own debug variables;
1647  see individual Task documentation.
1648 
1649  Examples
1650  --------
1651  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1652  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1653  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1654  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1655  and filter to be coadded (specified using
1656  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1657  along with a list of coaddTempExps to attempt to coadd (specified using
1658  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1659  Only the warps that cover the specified tract and patch will be coadded.
1660  A list of the available optional arguments can be obtained by calling
1661  ``assembleCoadd.py`` with the ``--help`` command line argument:
1662 
1663  .. code-block:: none
1664 
1665  assembleCoadd.py --help
1666 
1667  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1668  context of multi-band processing, we will generate the HSC-I & -R band
1669  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1670  To begin, assuming that the lsst stack has been already set up, we must
1671  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1672  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1673  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1674  directory. To begin assembling the coadds, we must first
1675 
1676  - processCcd
1677  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1678  - makeSkyMap
1679  create a skymap that covers the area of the sky present in the raw exposures
1680  - makeCoaddTempExp
1681  warp the individual calibrated exposures to the tangent plane of the coadd
1682 
1683  We can perform all of these steps by running
1684 
1685  .. code-block:: none
1686 
1687  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1688 
1689  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1690  warped data, we call ``assembleCoadd.py`` as follows:
1691 
1692  .. code-block:: none
1693 
1694  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1695  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1696  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1697  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1698  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1699  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1700  --selectId visit=903988 ccd=24
1701 
1702  This will process the HSC-I band data. The results are written in
1703  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1704  """
1705  ConfigClass = CompareWarpAssembleCoaddConfig
1706  _DefaultName = "compareWarpAssembleCoadd"
1707 
1708  def __init__(self, *args, **kwargs):
1709  AssembleCoaddTask.__init__(self, *args, **kwargs)
1710  self.makeSubtask("assembleStaticSkyModel")
1711  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1712  self.makeSubtask("detect", schema=detectionSchema)
1713  if self.config.doPreserveContainedBySource:
1714  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1715  if self.config.doScaleWarpVariance:
1716  self.makeSubtask("scaleWarpVariance")
1717 
1718  def makeSupplementaryData(self, dataRef, selectDataList):
1719  """Make inputs specific to Subclass.
1720 
1721  Generate a templateCoadd to use as a native model of static sky to
1722  subtract from warps.
1723 
1724  Parameters
1725  ----------
1726  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1727  Butler dataRef for supplementary data.
1728  selectDataList : `list`
1729  List of data references to Warps.
1730 
1731  Returns
1732  -------
1733  result : `lsst.pipe.base.Struct`
1734  Result struct with components:
1735 
1736  - ``templateCoaddcoadd``: coadded exposure (``lsst.afw.image.Exposure``).
1737  """
1738  templateCoadd = self.assembleStaticSkyModel.run(dataRef, selectDataList)
1739 
1740  if templateCoadd is None:
1741  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1742  self.assembleStaticSkyModel.warpType[1:])
1743  message = """No %(warpName)s warps were found to build the template coadd which is
1744  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1745  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1746  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1747 
1748  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1749  another algorithm like:
1750 
1751  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1752  config.assemble.retarget(SafeClipAssembleCoaddTask)
1753  """ % {"warpName": warpName}
1754  raise RuntimeError(message)
1755 
1756  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1757 
1758  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1759  supplementaryData, *args, **kwargs):
1760  """Assemble the coadd.
1761 
1762  Find artifacts and apply them to the warps' masks creating a list of
1763  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
1764  plane. Then pass these alternative masks to the base class's assemble
1765  method.
1766 
1767  Parameters
1768  ----------
1769  skyInfo : `lsst.pipe.base.Struct`
1770  Patch geometry information.
1771  tempExpRefList : `list`
1772  List of data references to warps.
1773  imageScalerList : `list`
1774  List of image scalers.
1775  weightList : `list`
1776  List of weights.
1777  supplementaryData : `lsst.pipe.base.Struct`
1778  This Struct must contain a ``templateCoadd`` that serves as the
1779  model of the static sky.
1780 
1781  Returns
1782  -------
1783  result : `lsst.pipe.base.Struct`
1784  Result struct with components:
1785 
1786  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1787  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
1788  """
1789  templateCoadd = supplementaryData.templateCoadd
1790  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1791  badMaskPlanes = self.config.badMaskPlanes[:]
1792  badMaskPlanes.append("CLIPPED")
1793  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1794 
1795  result = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1796  spanSetMaskList, mask=badPixelMask)
1797 
1798  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
1799  # Psf-Matching moves the real edge inwards
1800  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1801  return result
1802 
1803  def applyAltEdgeMask(self, mask, altMaskList):
1804  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1805 
1806  Parameters
1807  ----------
1808  mask : `lsst.afw.image.Mask`
1809  Original mask.
1810  altMaskList : `list`
1811  List of Dicts containing ``spanSet`` lists.
1812  Each element contains the new mask plane name (e.g. "CLIPPED
1813  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
1814  the mask.
1815  """
1816  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
1817  for visitMask in altMaskList:
1818  if "EDGE" in visitMask:
1819  for spanSet in visitMask['EDGE']:
1820  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1821 
1822  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1823  """Find artifacts.
1824 
1825  Loop through warps twice. The first loop builds a map with the count
1826  of how many epochs each pixel deviates from the templateCoadd by more
1827  than ``config.chiThreshold`` sigma. The second loop takes each
1828  difference image and filters the artifacts detected in each using
1829  count map to filter out variable sources and sources that are
1830  difficult to subtract cleanly.
1831 
1832  Parameters
1833  ----------
1834  templateCoadd : `lsst.afw.image.Exposure`
1835  Exposure to serve as model of static sky.
1836  tempExpRefList : `list`
1837  List of data references to warps.
1838  imageScalerList : `list`
1839  List of image scalers.
1840 
1841  Returns
1842  -------
1843  altMasks : `list`
1844  List of dicts containing information about CLIPPED
1845  (i.e., artifacts), NO_DATA, and EDGE pixels.
1846  """
1847 
1848  self.log.debug("Generating Count Image, and mask lists.")
1849  coaddBBox = templateCoadd.getBBox()
1850  slateIm = afwImage.ImageU(coaddBBox)
1851  epochCountImage = afwImage.ImageU(coaddBBox)
1852  nImage = afwImage.ImageU(coaddBBox)
1853  spanSetArtifactList = []
1854  spanSetNoDataMaskList = []
1855  spanSetEdgeList = []
1856  badPixelMask = self.getBadPixelMask()
1857 
1858  # mask of the warp diffs should = that of only the warp
1859  templateCoadd.mask.clearAllMaskPlanes()
1860 
1861  if self.config.doPreserveContainedBySource:
1862  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1863  else:
1864  templateFootprints = None
1865 
1866  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1867  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1868  if warpDiffExp is not None:
1869  # This nImage only approximates the final nImage because it uses the PSF-matched mask
1870  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1871  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1872  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1873  fpSet.positive.merge(fpSet.negative)
1874  footprints = fpSet.positive
1875  slateIm.set(0)
1876  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1877 
1878  # Remove artifacts due to defects before they contribute to the epochCountImage
1879  if self.config.doPrefilterArtifacts:
1880  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
1881  for spans in spanSetList:
1882  spans.setImage(slateIm, 1, doClip=True)
1883  epochCountImage += slateIm
1884 
1885  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1886  # undergo a second convolution. Pixels with data in the direct warp
1887  # but not in the PSF-matched warp will not have their artifacts detected.
1888  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1889  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1890  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1891  nansMask.setXY0(warpDiffExp.getXY0())
1892  edgeMask = warpDiffExp.mask
1893  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1894  edgeMask.getPlaneBitMask("EDGE")).split()
1895  else:
1896  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1897  # In this case, mask the whole epoch
1898  nansMask = afwImage.MaskX(coaddBBox, 1)
1899  spanSetList = []
1900  spanSetEdgeMask = []
1901 
1902  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1903 
1904  spanSetNoDataMaskList.append(spanSetNoDataMask)
1905  spanSetArtifactList.append(spanSetList)
1906  spanSetEdgeList.append(spanSetEdgeMask)
1907 
1908  if lsstDebug.Info(__name__).saveCountIm:
1909  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1910  epochCountImage.writeFits(path)
1911 
1912  for i, spanSetList in enumerate(spanSetArtifactList):
1913  if spanSetList:
1914  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
1915  templateFootprints)
1916  spanSetArtifactList[i] = filteredSpanSetList
1917 
1918  altMasks = []
1919  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1920  altMasks.append({'CLIPPED': artifacts,
1921  'NO_DATA': noData,
1922  'EDGE': edge})
1923  return altMasks
1924 
1925  def prefilterArtifacts(self, spanSetList, exp):
1926  """Remove artifact candidates covered by bad mask plane.
1927 
1928  Any future editing of the candidate list that does not depend on
1929  temporal information should go in this method.
1930 
1931  Parameters
1932  ----------
1933  spanSetList : `list`
1934  List of SpanSets representing artifact candidates.
1935  exp : `lsst.afw.image.Exposure`
1936  Exposure containing mask planes used to prefilter.
1937 
1938  Returns
1939  -------
1940  returnSpanSetList : `list`
1941  List of SpanSets with artifacts.
1942  """
1943  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1944  goodArr = (exp.mask.array & badPixelMask) == 0
1945  returnSpanSetList = []
1946  bbox = exp.getBBox()
1947  x0, y0 = exp.getXY0()
1948  for i, span in enumerate(spanSetList):
1949  y, x = span.clippedTo(bbox).indices()
1950  yIndexLocal = numpy.array(y) - y0
1951  xIndexLocal = numpy.array(x) - x0
1952  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1953  if goodRatio > self.config.prefilterArtifactsRatio:
1954  returnSpanSetList.append(span)
1955  return returnSpanSetList
1956 
1957  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1958  """Filter artifact candidates.
1959 
1960  Parameters
1961  ----------
1962  spanSetList : `list`
1963  List of SpanSets representing artifact candidates.
1964  epochCountImage : `lsst.afw.image.Image`
1965  Image of accumulated number of warpDiff detections.
1966  nImage : `lsst.afw.image.Image`
1967  Image of the accumulated number of total epochs contributing.
1968 
1969  Returns
1970  -------
1971  maskSpanSetList : `list`
1972  List of SpanSets with artifacts.
1973  """
1974 
1975  maskSpanSetList = []
1976  x0, y0 = epochCountImage.getXY0()
1977  for i, span in enumerate(spanSetList):
1978  y, x = span.indices()
1979  yIdxLocal = [y1 - y0 for y1 in y]
1980  xIdxLocal = [x1 - x0 for x1 in x]
1981  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1982  totalN = nImage.array[yIdxLocal, xIdxLocal]
1983 
1984  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
1985  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1986  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1987  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1988  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1989  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1990  (outlierN <= effectiveMaxNumEpochs))
1991  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1992  if percentBelowThreshold > self.config.spatialThreshold:
1993  maskSpanSetList.append(span)
1994 
1995  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
1996  # If a candidate is contained by a footprint on the template coadd, do not clip
1997  filteredMaskSpanSetList = []
1998  for span in maskSpanSetList:
1999  doKeep = True
2000  for footprint in footprintsToExclude.positive.getFootprints():
2001  if footprint.spans.contains(span):
2002  doKeep = False
2003  break
2004  if doKeep:
2005  filteredMaskSpanSetList.append(span)
2006  maskSpanSetList = filteredMaskSpanSetList
2007 
2008  return maskSpanSetList
2009 
2010  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2011  """Fetch a warp from the butler and return a warpDiff.
2012 
2013  Parameters
2014  ----------
2015  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2016  Butler dataRef for the warp.
2017  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2018  An image scaler object.
2019  templateCoadd : `lsst.afw.image.Exposure`
2020  Exposure to be substracted from the scaled warp.
2021 
2022  Returns
2023  -------
2024  warp : `lsst.afw.image.Exposure`
2025  Exposure of the image difference between the warp and template.
2026  """
2027 
2028  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2029  warpName = self.getTempExpDatasetName('psfMatched')
2030  if not warpRef.datasetExists(warpName):
2031  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2032  return None
2033  warp = warpRef.get(warpName, immediate=True)
2034  # direct image scaler OK for PSF-matched Warp
2035  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2036  mi = warp.getMaskedImage()
2037  if self.config.doScaleWarpVariance:
2038  try:
2039  self.scaleWarpVariance.run(mi)
2040  except Exception as exc:
2041  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2042  mi -= templateCoadd.getMaskedImage()
2043  return warp
2044 
2045  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2046  """Return a path to which to write debugging output.
2047 
2048  Creates a hyphen-delimited string of dataId values for simple filenames.
2049 
2050  Parameters
2051  ----------
2052  prefix : `str`
2053  Prefix for filename.
2054  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2055  Butler dataRef to make the path from.
2056  coaddLevel : `bool`, optional.
2057  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2058  'filter', but no 'visit').
2059 
2060  Returns
2061  -------
2062  result : `str`
2063  Path for debugging output.
2064  """
2065  if coaddLevel:
2066  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2067  else:
2068  keys = warpRef.dataId.keys()
2069  keyList = sorted(keys, reverse=True)
2070  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2071  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2072  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:171
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:94
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def makeSupplementaryData(self, dataRef, selectDataList)
def getTempExpRefList(self, patchRef, calExpRefList)
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, dataRef, selectDataList=[])
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:127
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:221
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:107
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)