lsst.pipe.tasks  16.0-18-g95848a16+4
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy
24 import lsst.pex.config as pexConfig
25 import lsst.pex.exceptions as pexExceptions
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.math as afwMath
29 import lsst.afw.table as afwTable
30 import lsst.afw.detection as afwDet
31 import lsst.coadd.utils as coaddUtils
32 import lsst.pipe.base as pipeBase
33 import lsst.meas.algorithms as measAlg
34 import lsst.log as log
35 import lsstDebug
36 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
37 from .interpImage import InterpImageTask
38 from .scaleZeroPoint import ScaleZeroPointTask
39 from .coaddHelpers import groupPatchExposures, getGroupDataRef
40 from .scaleVariance import ScaleVarianceTask
41 from lsst.meas.algorithms import SourceDetectionTask
42 
43 __all__ = ["AssembleCoaddTask", "AssembleCoaddConfig", "SafeClipAssembleCoaddTask",
44  "SafeClipAssembleCoaddConfig", "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
45 
46 
47 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
48  """Configuration parameters for the `AssembleCoaddTask`.
49 
50  Notes
51  -----
52  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
53  only set the bitplane config.brightObjectMaskName. To make this useful you
54  *must* also configure the flags.pixel algorithm, for example by adding
55 
56  .. code-block:: none
57 
58  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
59  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
60 
61  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
62  """
63  warpType = pexConfig.Field(
64  doc="Warp name: one of 'direct' or 'psfMatched'",
65  dtype=str,
66  default="direct",
67  )
68  subregionSize = pexConfig.ListField(
69  dtype=int,
70  doc="Width, height of stack subregion size; "
71  "make small enough that a full stack of images will fit into memory at once.",
72  length=2,
73  default=(2000, 2000),
74  )
75  statistic = pexConfig.Field(
76  dtype=str,
77  doc="Main stacking statistic for aggregating over the epochs.",
78  default="MEANCLIP",
79  )
80  doSigmaClip = pexConfig.Field(
81  dtype=bool,
82  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
83  default=False,
84  )
85  sigmaClip = pexConfig.Field(
86  dtype=float,
87  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
88  default=3.0,
89  )
90  clipIter = pexConfig.Field(
91  dtype=int,
92  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
93  default=2,
94  )
95  calcErrorFromInputVariance = pexConfig.Field(
96  dtype=bool,
97  doc="Calculate coadd variance from input variance by stacking statistic."
98  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
99  default=True,
100  )
101  scaleZeroPoint = pexConfig.ConfigurableField(
102  target=ScaleZeroPointTask,
103  doc="Task to adjust the photometric zero point of the coadd temp exposures",
104  )
105  doInterp = pexConfig.Field(
106  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
107  dtype=bool,
108  default=True,
109  )
110  interpImage = pexConfig.ConfigurableField(
111  target=InterpImageTask,
112  doc="Task to interpolate (and extrapolate) over NaN pixels",
113  )
114  doWrite = pexConfig.Field(
115  doc="Persist coadd?",
116  dtype=bool,
117  default=True,
118  )
119  doNImage = pexConfig.Field(
120  doc="Create image of number of contributing exposures for each pixel",
121  dtype=bool,
122  default=False,
123  )
124  doUsePsfMatchedPolygons = pexConfig.Field(
125  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
126  dtype=bool,
127  default=False,
128  )
129  maskPropagationThresholds = pexConfig.DictField(
130  keytype=str,
131  itemtype=float,
132  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
133  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
134  "would have contributed exceeds this value."),
135  default={"SAT": 0.1},
136  )
137  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
138  doc="Mask planes to remove before coadding")
139  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
140  doc="Set mask and flag bits for bright objects?")
141  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
142  doc="Name of mask bit used for bright objects")
143  coaddPsf = pexConfig.ConfigField(
144  doc="Configuration for CoaddPsf",
145  dtype=measAlg.CoaddPsfConfig,
146  )
147  doAttachTransmissionCurve = pexConfig.Field(
148  dtype=bool, default=False, optional=False,
149  doc=("Attach a piecewise TransmissionCurve for the coadd? "
150  "(requires all input Exposures to have TransmissionCurves).")
151  )
152 
153  def setDefaults(self):
154  CoaddBaseTask.ConfigClass.setDefaults(self)
155  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
156 
157  def validate(self):
158  CoaddBaseTask.ConfigClass.validate(self)
159  if self.doPsfMatch:
160  # Backwards compatibility.
161  # Configs do not have loggers
162  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
163  self.warpType = 'psfMatched'
164  if self.doSigmaClip and self.statistic != "MEANCLIP":
165  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
166  self.statistic = "MEANCLIP"
167  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
168  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
169  "compute and set a non-zero coadd variance estimate." % (self.statistic))
170 
171  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
172  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
173  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
174  if str(k) not in unstackableStats]
175  raise ValueError("statistic %s is not allowed. Please choose one of %s."
176  % (self.statistic, stackableStats))
177 
178 
180  """Assemble a coadded image from a set of warps (coadded temporary exposures).
181 
182  We want to assemble a coadded image from a set of Warps (also called
183  coadded temporary exposures or ``coaddTempExps``).
184  Each input Warp covers a patch on the sky and corresponds to a single
185  run/visit/exposure of the covered patch. We provide the task with a list
186  of Warps (``selectDataList``) from which it selects Warps that cover the
187  specified patch (pointed at by ``dataRef``).
188  Each Warp that goes into a coadd will typically have an independent
189  photometric zero-point. Therefore, we must scale each Warp to set it to
190  a common photometric zeropoint. WarpType may be one of 'direct' or
191  'psfMatched', and the boolean configs `config.makeDirect` and
192  `config.makePsfMatched` set which of the warp types will be coadded.
193  The coadd is computed as a mean with optional outlier rejection.
194  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
195  Finally, Warps can have bad 'NaN' pixels which received no input from the
196  source calExps. We interpolate over these bad (NaN) pixels.
197 
198  `AssembleCoaddTask` uses several sub-tasks. These are
199 
200  - `ScaleZeroPointTask`
201  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
202  - `InterpImageTask`
203  - interpolate across bad pixels (NaN) in the final coadd
204 
205  You can retarget these subtasks if you wish.
206 
207  Notes
208  -----
209  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
210  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
211  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
212  no debug variables of its own. Some of the subtasks may support debug
213  variables. See the documentation for the subtasks for further information.
214 
215  Examples
216  --------
217  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
218  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
219  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
220  inputs: a data reference to the tract patch and filter to be coadded, and
221  a list of Warps to attempt to coadd. These are specified using ``--id`` and
222  ``--selectId``, respectively:
223 
224  .. code-block:: none
225 
226  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
227  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
228 
229  Only the Warps that cover the specified tract and patch will be coadded.
230  A list of the available optional arguments can be obtained by calling
231  ``assembleCoadd.py`` with the ``--help`` command line argument:
232 
233  .. code-block:: none
234 
235  assembleCoadd.py --help
236 
237  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
238  multi-band processing, we will generate the HSC-I & -R band coadds from
239  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
240  assuming that the lsst stack has been already set up, we must set up the
241  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
242  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
243  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
244  coadds, we must first
245 
246  - processCcd
247  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
248  - makeSkyMap
249  - create a skymap that covers the area of the sky present in the raw exposures
250  - makeCoaddTempExp
251  - warp the individual calibrated exposures to the tangent plane of the coadd
252 
253  We can perform all of these steps by running
254 
255  .. code-block:: none
256 
257  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
258 
259  This will produce warped exposures for each visit. To coadd the warped
260  data, we call assembleCoadd.py as follows:
261 
262  .. code-block:: none
263 
264  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
265  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
266  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
267  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
268  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
269  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
270  --selectId visit=903988 ccd=24
271 
272  that will process the HSC-I band data. The results are written in
273  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
274 
275  You may also choose to run:
276 
277  .. code-block:: none
278 
279  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
280  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
281  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
282  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
283  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
284  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
285  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
286  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
287 
288  to generate the coadd for the HSC-R band if you are interested in
289  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
290  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
291  rather than `AssembleCoaddTask` to make the coadd.
292  """
293  ConfigClass = AssembleCoaddConfig
294  _DefaultName = "assembleCoadd"
295 
296  def __init__(self, *args, **kwargs):
297  CoaddBaseTask.__init__(self, *args, **kwargs)
298  self.makeSubtask("interpImage")
299  self.makeSubtask("scaleZeroPoint")
300 
301  if self.config.doMaskBrightObjects:
302  mask = afwImage.Mask()
303  try:
304  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
305  except pexExceptions.LsstCppException:
306  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
307  mask.getMaskPlaneDict().keys())
308  del mask
309 
310  self.warpType = self.config.warpType
311 
312  @pipeBase.timeMethod
313  def runDataRef(self, dataRef, selectDataList=[]):
314  """Assemble a coadd from a set of Warps.
315 
316  Coadd a set of Warps. Compute weights to be applied to each Warp and
317  find scalings to match the photometric zeropoint to a reference Warp.
318  Assemble the Warps using `run`. Interpolate over NaNs and
319  optionally write the coadd to disk. Return the coadded exposure.
320 
321  Parameters
322  ----------
323  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
324  Data reference defining the patch for coaddition and the
325  reference Warp (if ``config.autoReference=False``).
326  Used to access the following data products:
327  - ``self.config.coaddName + "Coadd_skyMap"``
328  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
329  - ``self.config.coaddName + "Coadd"``
330  selectDataList : `list`
331  List of data references to Warps. Data to be coadded will be
332  selected from this list based on overlap with the patch defined
333  by dataRef.
334 
335  Returns
336  -------
337  retStruct : `lsst.pipe.base.Struct`
338  Result struct with components:
339 
340  - ``coaddExposure``: coadded exposure (``Exposure``).
341  - ``nImage``: exposure count image (``Image``).
342  """
343  skyInfo = self.getSkyInfo(dataRef)
344  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
345  if len(calExpRefList) == 0:
346  self.log.warn("No exposures to coadd")
347  return
348  self.log.info("Coadding %d exposures", len(calExpRefList))
349 
350  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
351  inputData = self.prepareInputs(tempExpRefList)
352  self.log.info("Found %d %s", len(inputData.tempExpRefList),
353  self.getTempExpDatasetName(self.warpType))
354  if len(inputData.tempExpRefList) == 0:
355  self.log.warn("No coadd temporary exposures found")
356  return
357 
358  supplementaryData = self.makeSupplementaryData(dataRef, selectDataList)
359 
360  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
361  inputData.weightList, supplementaryData=supplementaryData)
362 
363  self.processResults(retStruct.coaddExposure, dataRef)
364  if self.config.doWrite:
365  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
366  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
367  if self.config.doNImage and retStruct.nImage is not None:
368  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
369 
370  return retStruct
371 
372  def processResults(self, coaddExposure, dataRef):
373  """Interpolate over missing data and mask bright stars.
374 
375  Parameters
376  ----------
377  coaddExposure : `lsst.afw.image.Exposure`
378  The coadded exposure to process.
379  dataRef : `lsst.daf.persistence.ButlerDataRef`
380  Butler data reference for supplementary data.
381  """
382  if self.config.doInterp:
383  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
384  # The variance must be positive; work around for DM-3201.
385  varArray = coaddExposure.variance.array
386  with numpy.errstate(invalid="ignore"):
387  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
388 
389  if self.config.doMaskBrightObjects:
390  brightObjectMasks = self.readBrightObjectMasks(dataRef)
391  self.setBrightObjectMasks(coaddExposure, dataRef.dataId, brightObjectMasks)
392 
393  def makeSupplementaryData(self, dataRef, selectDataList):
394  """Make additional inputs to run() specific to subclasses.
395 
396  Available to be implemented by subclasses only if they need the
397  coadd dataRef for performing preliminary processing before
398  assembling the coadd.
399 
400  Parameters
401  ----------
402  dataRef : `lsst.daf.persistence.ButlerDataRef`
403  Butler data reference for supplementary data.
404  selectDataList : `list`
405  List of data references to Warps.
406  """
407  pass
408 
409  def getTempExpRefList(self, patchRef, calExpRefList):
410  """Generate list data references corresponding to warped exposures
411  that lie within the patch to be coadded.
412 
413  Parameters
414  ----------
415  patchRef : `dataRef`
416  Data reference for patch.
417  calExpRefList : `list`
418  List of data references for input calexps.
419 
420  Returns
421  -------
422  tempExpRefList : `list`
423  List of Warp/CoaddTempExp data references.
424  """
425  butler = patchRef.getButler()
426  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
427  self.getTempExpDatasetName(self.warpType))
428  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
429  g, groupData.keys) for
430  g in groupData.groups.keys()]
431  return tempExpRefList
432 
433  def prepareInputs(self, refList):
434  """Prepare the input warps for coaddition by measuring the weight for
435  each warp and the scaling for the photometric zero point.
436 
437  Each Warp has its own photometric zeropoint and background variance.
438  Before coadding these Warps together, compute a scale factor to
439  normalize the photometric zeropoint and compute the weight for each Warp.
440 
441  Parameters
442  ----------
443  refList : `list`
444  List of data references to tempExp
445 
446  Returns
447  -------
448  result : `lsst.pipe.base.Struct`
449  Result struct with components:
450 
451  - ``tempExprefList``: `list` of data references to tempExp.
452  - ``weightList``: `list` of weightings.
453  - ``imageScalerList``: `list` of image scalers.
454  """
455  statsCtrl = afwMath.StatisticsControl()
456  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
457  statsCtrl.setNumIter(self.config.clipIter)
458  statsCtrl.setAndMask(self.getBadPixelMask())
459  statsCtrl.setNanSafe(True)
460  # compute tempExpRefList: a list of tempExpRef that actually exist
461  # and weightList: a list of the weight of the associated coadd tempExp
462  # and imageScalerList: a list of scale factors for the associated coadd tempExp
463  tempExpRefList = []
464  weightList = []
465  imageScalerList = []
466  tempExpName = self.getTempExpDatasetName(self.warpType)
467  for tempExpRef in refList:
468  if not tempExpRef.datasetExists(tempExpName):
469  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
470  continue
471 
472  tempExp = tempExpRef.get(tempExpName, immediate=True)
473  maskedImage = tempExp.getMaskedImage()
474  imageScaler = self.scaleZeroPoint.computeImageScaler(
475  exposure=tempExp,
476  dataRef=tempExpRef,
477  )
478  try:
479  imageScaler.scaleMaskedImage(maskedImage)
480  except Exception as e:
481  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
482  continue
483  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
484  afwMath.MEANCLIP, statsCtrl)
485  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
486  weight = 1.0 / float(meanVar)
487  if not numpy.isfinite(weight):
488  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
489  continue
490  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
491 
492  del maskedImage
493  del tempExp
494 
495  tempExpRefList.append(tempExpRef)
496  weightList.append(weight)
497  imageScalerList.append(imageScaler)
498 
499  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
500  imageScalerList=imageScalerList)
501 
502  def prepareStats(self, mask=None):
503  """Prepare the statistics for coadding images.
504 
505  Parameters
506  ----------
507  mask : `int`, optional
508  Bit mask value to exclude from coaddition.
509 
510  Returns
511  -------
512  stats : `lsst.pipe.base.Struct`
513  Statistics structure with the following fields:
514 
515  - ``statsCtrl``: Statistics control object for coadd
516  (`lsst.afw.math.StatisticsControl`)
517  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
518  """
519  if mask is None:
520  mask = self.getBadPixelMask()
521  statsCtrl = afwMath.StatisticsControl()
522  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
523  statsCtrl.setNumIter(self.config.clipIter)
524  statsCtrl.setAndMask(mask)
525  statsCtrl.setNanSafe(True)
526  statsCtrl.setWeighted(True)
527  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
528  for plane, threshold in self.config.maskPropagationThresholds.items():
529  bit = afwImage.Mask.getMaskPlane(plane)
530  statsCtrl.setMaskPropagationThreshold(bit, threshold)
531  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
532  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
533 
534  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
535  altMaskList=None, mask=None, supplementaryData=None):
536  """Assemble a coadd from input warps
537 
538  Assemble the coadd using the provided list of coaddTempExps. Since
539  the full coadd covers a patch (a large area), the assembly is
540  performed over small areas on the image at a time in order to
541  conserve memory usage. Iterate over subregions within the outer
542  bbox of the patch using `assembleSubregion` to stack the corresponding
543  subregions from the coaddTempExps with the statistic specified.
544  Set the edge bits the coadd mask based on the weight map.
545 
546  Parameters
547  ----------
548  skyInfo : `lsst.pipe.base.Struct`
549  Struct with geometric information about the patch.
550  tempExpRefList : `list`
551  List of data references to Warps (previously called CoaddTempExps).
552  imageScalerList : `list`
553  List of image scalers.
554  weightList : `list`
555  List of weights
556  altMaskList : `list`, optional
557  List of alternate masks to use rather than those stored with
558  tempExp.
559  mask : `lsst.afw.image.Mask`, optional
560  Mask to ignore when coadding
561  supplementaryData : lsst.pipe.base.Struct, optional
562  Struct with additional data products needed to assemble coadd.
563  Only used by subclasses that implement `makeSupplementaryData`
564  and override `run`.
565 
566  Returns
567  -------
568  result : `lsst.pipe.base.Struct`
569  Result struct with components:
570 
571  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
572  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
573  """
574  tempExpName = self.getTempExpDatasetName(self.warpType)
575  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
576  stats = self.prepareStats(mask=mask)
577 
578  if altMaskList is None:
579  altMaskList = [None]*len(tempExpRefList)
580 
581  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
582  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
583  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
584  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
585  coaddMaskedImage = coaddExposure.getMaskedImage()
586  subregionSizeArr = self.config.subregionSize
587  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
588  # if nImage is requested, create a zero one which can be passed to assembleSubregion
589  if self.config.doNImage:
590  nImage = afwImage.ImageU(skyInfo.bbox)
591  else:
592  nImage = None
593  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
594  try:
595  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
596  weightList, altMaskList, stats.flags, stats.ctrl,
597  nImage=nImage)
598  except Exception as e:
599  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
600 
601  self.setInexactPsf(coaddMaskedImage.getMask())
602  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
603  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
604  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
605  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
606 
607  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
608  """Set the metadata for the coadd.
609 
610  This basic implementation sets the filter from the first input.
611 
612  Parameters
613  ----------
614  coaddExposure : `lsst.afw.image.Exposure`
615  The target exposure for the coadd.
616  tempExpRefList : `list`
617  List of data references to tempExp.
618  weightList : `list`
619  List of weights.
620  """
621  assert len(tempExpRefList) == len(weightList), "Length mismatch"
622  tempExpName = self.getTempExpDatasetName(self.warpType)
623  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
624  # (and we need more than just the PropertySet that contains the header), which is not possible
625  # with the current butler (see #2777).
626  tempExpList = [tempExpRef.get(tempExpName + "_sub",
627  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
628  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
629  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
630 
631  coaddExposure.setFilter(tempExpList[0].getFilter())
632  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
633  coaddInputs.ccds.reserve(numCcds)
634  coaddInputs.visits.reserve(len(tempExpList))
635 
636  for tempExp, weight in zip(tempExpList, weightList):
637  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
638 
639  if self.config.doUsePsfMatchedPolygons:
640  self.shrinkValidPolygons(coaddInputs)
641 
642  coaddInputs.visits.sort()
643  if self.warpType == "psfMatched":
644  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
645  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
646  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
647  # having the maximum width (sufficient because square)
648  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
649  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
650  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
651  else:
652  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
653  self.config.coaddPsf.makeControl())
654  coaddExposure.setPsf(psf)
655  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
656  coaddExposure.getWcs())
657  coaddExposure.getInfo().setApCorrMap(apCorrMap)
658  if self.config.doAttachTransmissionCurve:
659  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
660  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
661 
662  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
663  altMaskList, statsFlags, statsCtrl, nImage=None):
664  """Assemble the coadd for a sub-region.
665 
666  For each coaddTempExp, check for (and swap in) an alternative mask
667  if one is passed. Remove mask planes listed in
668  `config.removeMaskPlanes`. Finally, stack the actual exposures using
669  `lsst.afw.math.statisticsStack` with the statistic specified by
670  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
671  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
672  an N-sigma clipped mean where N and iterations are specified by
673  statsCtrl. Assign the stacked subregion back to the coadd.
674 
675  Parameters
676  ----------
677  coaddExposure : `lsst.afw.image.Exposure`
678  The target exposure for the coadd.
679  bbox : `lsst.afw.geom.Box`
680  Sub-region to coadd.
681  tempExpRefList : `list`
682  List of data reference to tempExp.
683  imageScalerList : `list`
684  List of image scalers.
685  weightList : `list`
686  List of weights.
687  altMaskList : `list`
688  List of alternate masks to use rather than those stored with
689  tempExp, or None. Each element is dict with keys = mask plane
690  name to which to add the spans.
691  statsFlags : `lsst.afw.math.Property`
692  Property object for statistic for coadd.
693  statsCtrl : `lsst.afw.math.StatisticsControl`
694  Statistics control object for coadd.
695  nImage : `lsst.afw.image.ImageU`, optional
696  Keeps track of exposure count for each pixel.
697  """
698  self.log.debug("Computing coadd over %s", bbox)
699  tempExpName = self.getTempExpDatasetName(self.warpType)
700  coaddExposure.mask.addMaskPlane("REJECTED")
701  coaddExposure.mask.addMaskPlane("CLIPPED")
702  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
703  maskMap = self.setRejectedMaskMapping(statsCtrl)
704  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
705  maskedImageList = []
706  if nImage is not None:
707  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
708  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
709  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
710  maskedImage = exposure.getMaskedImage()
711  mask = maskedImage.getMask()
712  if altMask is not None:
713  self.applyAltMaskPlanes(mask, altMask)
714  imageScaler.scaleMaskedImage(maskedImage)
715 
716  # Add 1 for each pixel which is not excluded by the exclude mask.
717  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
718  if nImage is not None:
719  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
720  if self.config.removeMaskPlanes:
721  self.removeMaskPlanes(maskedImage)
722  maskedImageList.append(maskedImage)
723 
724  with self.timer("stack"):
725  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
726  clipped, # also set output to CLIPPED if sigma-clipped
727  maskMap)
728  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
729  if nImage is not None:
730  nImage.assign(subNImage, bbox)
731 
732  def removeMaskPlanes(self, maskedImage):
733  """Unset the mask of an image for mask planes specified in the config.
734 
735  Parameters
736  ----------
737  maskedImage : `lsst.afw.image.MaskedImage`
738  The masked image to be modified.
739  """
740  mask = maskedImage.getMask()
741  for maskPlane in self.config.removeMaskPlanes:
742  try:
743  mask &= ~mask.getPlaneBitMask(maskPlane)
744  except Exception as e:
745  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
746 
747  @staticmethod
748  def setRejectedMaskMapping(statsCtrl):
749  """Map certain mask planes of the warps to new planes for the coadd.
750 
751  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
752  or CLIPPED, set it to REJECTED on the coadd.
753  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
754  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
755 
756  Parameters
757  ----------
758  statsCtrl : `lsst.afw.math.StatisticsControl`
759  Statistics control object for coadd
760 
761  Returns
762  -------
763  maskMap : `list` of `tuple` of `int`
764  A list of mappings of mask planes of the warped exposures to
765  mask planes of the coadd.
766  """
767  edge = afwImage.Mask.getPlaneBitMask("EDGE")
768  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
769  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
770  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
771  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
772  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
773  (clipped, clipped)]
774  return maskMap
775 
776  def applyAltMaskPlanes(self, mask, altMaskSpans):
777  """Apply in place alt mask formatted as SpanSets to a mask.
778 
779  Parameters
780  ----------
781  mask : `lsst.afw.image.Mask`
782  Original mask.
783  altMaskSpans : `dict`
784  SpanSet lists to apply. Each element contains the new mask
785  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
786  and list of SpanSets to apply to the mask.
787 
788  Returns
789  -------
790  mask : `lsst.afw.image.Mask`
791  Updated mask.
792  """
793  if self.config.doUsePsfMatchedPolygons:
794  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
795  # Clear away any other masks outside the validPolygons. These pixels are no longer
796  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
797  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
798  # This mask-clearing step must only occur *before* applying the new masks below
799  for spanSet in altMaskSpans['NO_DATA']:
800  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
801 
802  for plane, spanSetList in altMaskSpans.items():
803  maskClipValue = mask.addMaskPlane(plane)
804  for spanSet in spanSetList:
805  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
806  return mask
807 
808  def shrinkValidPolygons(self, coaddInputs):
809  """Shrink coaddInputs' ccds' ValidPolygons in place.
810 
811  Either modify each ccd's validPolygon in place, or if CoaddInputs
812  does not have a validPolygon, create one from its bbox.
813 
814  Parameters
815  ----------
816  coaddInputs : `lsst.afw.image.coaddInputs`
817  Original mask.
818 
819  """
820  for ccd in coaddInputs.ccds:
821  polyOrig = ccd.getValidPolygon()
822  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
823  validPolyBBox.grow(-self.config.matchingKernelSize//2)
824  if polyOrig:
825  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
826  else:
827  validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
828  ccd.setValidPolygon(validPolygon)
829 
830  def readBrightObjectMasks(self, dataRef):
831  """Retrieve the bright object masks.
832 
833  Returns None on failure.
834 
835  Parameters
836  ----------
837  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
838  A Butler dataRef.
839 
840  Returns
841  -------
842  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
843  Bright object mask from the Butler object, or None if it cannot
844  be retrieved.
845  """
846  try:
847  return dataRef.get("brightObjectMask", immediate=True)
848  except Exception as e:
849  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
850  return None
851 
852  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
853  """Set the bright object masks.
854 
855  Parameters
856  ----------
857  exposure : `lsst.afw.image.Exposure`
858  Exposure under consideration.
859  dataId : `lsst.daf.persistence.dataId`
860  Data identifier dict for patch.
861  brightObjectMasks : `lsst.afw.table`
862  Table of bright objects to mask.
863  """
864  #
865  # Check the metadata specifying the tract/patch/filter
866  #
867  if brightObjectMasks is None:
868  self.log.warn("Unable to apply bright object mask: none supplied")
869  return
870  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
871  md = brightObjectMasks.table.getMetadata()
872  for k in dataId:
873  if not md.exists(k):
874  self.log.warn("Expected to see %s in metadata", k)
875  else:
876  if md.getScalar(k) != dataId[k]:
877  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
878 
879  mask = exposure.getMaskedImage().getMask()
880  wcs = exposure.getWcs()
881  plateScale = wcs.getPixelScale().asArcseconds()
882 
883  for rec in brightObjectMasks:
884  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
885  if rec["type"] == "box":
886  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
887  width = rec["width"].asArcseconds()/plateScale # convert to pixels
888  height = rec["height"].asArcseconds()/plateScale # convert to pixels
889 
890  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
891  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
892 
893  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
894  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
895  spans = afwGeom.SpanSet(bbox)
896  elif rec["type"] == "circle":
897  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
898  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
899  else:
900  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
901  continue
902  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
903 
904  def setInexactPsf(self, mask):
905  """Set INEXACT_PSF mask plane.
906 
907  If any of the input images isn't represented in the coadd (due to
908  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
909  these pixels.
910 
911  Parameters
912  ----------
913  mask : `lsst.afw.image.Mask`
914  Coadded exposure's mask, modified in-place.
915  """
916  mask.addMaskPlane("INEXACT_PSF")
917  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
918  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
919  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
920  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
921  array = mask.getArray()
922  selected = array & (sensorEdge | clipped | rejected) > 0
923  array[selected] |= inexactPsf
924 
925  @classmethod
926  def _makeArgumentParser(cls):
927  """Create an argument parser.
928  """
929  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
930  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
931  cls.ConfigClass().warpType + "Warp",
932  help="data ID, e.g. --id tract=12345 patch=1,2",
933  ContainerClass=AssembleCoaddDataIdContainer)
934  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
935  ContainerClass=SelectDataIdContainer)
936  return parser
937 
938  @staticmethod
939  def _subBBoxIter(bbox, subregionSize):
940  """Iterate over subregions of a bbox.
941 
942  Parameters
943  ----------
944  bbox : `lsst.afw.geom.Box2I`
945  Bounding box over which to iterate.
946  subregionSize: `lsst.afw.geom.Extent2I`
947  Size of sub-bboxes.
948 
949  Yields
950  ------
951  subBBox : `lsst.afw.geom.Box2I`
952  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
953  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
954  the edges of ``bbox``, but it will never be empty.
955  """
956  if bbox.isEmpty():
957  raise RuntimeError("bbox %s is empty" % (bbox,))
958  if subregionSize[0] < 1 or subregionSize[1] < 1:
959  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
960 
961  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
962  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
963  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
964  subBBox.clip(bbox)
965  if subBBox.isEmpty():
966  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
967  "colShift=%s, rowShift=%s" %
968  (bbox, subregionSize, colShift, rowShift))
969  yield subBBox
970 
971 
972 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
973  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
974  """
975 
976  def makeDataRefList(self, namespace):
977  """Make self.refList from self.idList.
978 
979  Parameters
980  ----------
981  namespace
982  Results of parsing command-line (with ``butler`` and ``log`` elements).
983  """
984  datasetType = namespace.config.coaddName + "Coadd"
985  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
986 
987  for dataId in self.idList:
988  # tract and patch are required
989  for key in keysCoadd:
990  if key not in dataId:
991  raise RuntimeError("--id must include " + key)
992 
993  dataRef = namespace.butler.dataRef(
994  datasetType=datasetType,
995  dataId=dataId,
996  )
997  self.refList.append(dataRef)
998 
999 
1000 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1001  """Function to count the number of pixels with a specific mask in a
1002  footprint.
1003 
1004  Find the intersection of mask & footprint. Count all pixels in the mask
1005  that are in the intersection that have bitmask set but do not have
1006  ignoreMask set. Return the count.
1007 
1008  Parameters
1009  ----------
1010  mask : `lsst.afw.image.Mask`
1011  Mask to define intersection region by.
1012  footprint : `lsst.afw.detection.Footprint`
1013  Footprint to define the intersection region by.
1014  bitmask
1015  Specific mask that we wish to count the number of occurances of.
1016  ignoreMask
1017  Pixels to not consider.
1018 
1019  Returns
1020  -------
1021  result : `int`
1022  Count of number of pixels in footprint with specified mask.
1023  """
1024  bbox = footprint.getBBox()
1025  bbox.clip(mask.getBBox(afwImage.PARENT))
1026  fp = afwImage.Mask(bbox)
1027  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1028  footprint.spans.setMask(fp, bitmask)
1029  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1030  (subMask.getArray() & ignoreMask) == 0).sum()
1031 
1032 
1034  """Configuration parameters for the SafeClipAssembleCoaddTask.
1035  """
1036  clipDetection = pexConfig.ConfigurableField(
1037  target=SourceDetectionTask,
1038  doc="Detect sources on difference between unclipped and clipped coadd")
1039  minClipFootOverlap = pexConfig.Field(
1040  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1041  dtype=float,
1042  default=0.6
1043  )
1044  minClipFootOverlapSingle = pexConfig.Field(
1045  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1046  "clipped when only one visit overlaps",
1047  dtype=float,
1048  default=0.5
1049  )
1050  minClipFootOverlapDouble = pexConfig.Field(
1051  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1052  "clipped when two visits overlap",
1053  dtype=float,
1054  default=0.45
1055  )
1056  maxClipFootOverlapDouble = pexConfig.Field(
1057  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1058  "considering two visits",
1059  dtype=float,
1060  default=0.15
1061  )
1062  minBigOverlap = pexConfig.Field(
1063  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1064  "when labeling clipped footprints",
1065  dtype=int,
1066  default=100
1067  )
1068 
1069  def setDefaults(self):
1070  """Set default values for clipDetection.
1071 
1072  Notes
1073  -----
1074  The numeric values for these configuration parameters were
1075  empirically determined, future work may further refine them.
1076  """
1077  AssembleCoaddConfig.setDefaults(self)
1078  self.clipDetection.doTempLocalBackground = False
1079  self.clipDetection.reEstimateBackground = False
1080  self.clipDetection.returnOriginalFootprints = False
1081  self.clipDetection.thresholdPolarity = "both"
1082  self.clipDetection.thresholdValue = 2
1083  self.clipDetection.nSigmaToGrow = 2
1084  self.clipDetection.minPixels = 4
1085  self.clipDetection.isotropicGrow = True
1086  self.clipDetection.thresholdType = "pixel_stdev"
1087  self.sigmaClip = 1.5
1088  self.clipIter = 3
1089  self.statistic = "MEAN"
1090 
1091  def validate(self):
1092  if self.doSigmaClip:
1093  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1094  "Ignoring doSigmaClip.")
1095  self.doSigmaClip = False
1096  if self.statistic != "MEAN":
1097  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1098  "(%s chosen). Please set statistic to MEAN."
1099  % (self.statistic))
1100  AssembleCoaddTask.ConfigClass.validate(self)
1101 
1102 
1104  """Assemble a coadded image from a set of coadded temporary exposures,
1105  being careful to clip & flag areas with potential artifacts.
1106 
1107  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1108  we clip outliers). The problem with doing this is that when computing the
1109  coadd PSF at a given location, individual visit PSFs from visits with
1110  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1111  In this task, we correct for this behavior by creating a new
1112  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1113  coaddTempExps and the final coadd where
1114 
1115  i. difference imaging suggests that there is an outlier and
1116  ii. this outlier appears on only one or two images.
1117 
1118  Such regions will not contribute to the final coadd. Furthermore, any
1119  routine to determine the coadd PSF can now be cognizant of clipped regions.
1120  Note that the algorithm implemented by this task is preliminary and works
1121  correctly for HSC data. Parameter modifications and or considerable
1122  redesigning of the algorithm is likley required for other surveys.
1123 
1124  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1125  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1126  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1127  if you wish.
1128 
1129  Notes
1130  -----
1131  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1132  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1133  see `baseDebug` for more about ``debug.py`` files.
1134  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1135  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1136  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1137  for further information.
1138 
1139  Examples
1140  --------
1141  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1142  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1143  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1144 
1145  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1146  and filter to be coadded (specified using
1147  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1148  along with a list of coaddTempExps to attempt to coadd (specified using
1149  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1150  Only the coaddTempExps that cover the specified tract and patch will be
1151  coadded. A list of the available optional arguments can be obtained by
1152  calling assembleCoadd.py with the --help command line argument:
1153 
1154  .. code-block:: none
1155 
1156  assembleCoadd.py --help
1157 
1158  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1159  context of multi-band processing, we will generate the HSC-I & -R band
1160  coadds from HSC engineering test data provided in the ci_hsc package.
1161  To begin, assuming that the lsst stack has been already set up, we must
1162  set up the obs_subaru and ci_hsc packages. This defines the environment
1163  variable $CI_HSC_DIR and points at the location of the package. The raw
1164  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1165  the coadds, we must first
1166 
1167  - ``processCcd``
1168  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1169  - ``makeSkyMap``
1170  create a skymap that covers the area of the sky present in the raw exposures
1171  - ``makeCoaddTempExp``
1172  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1173 
1174  We can perform all of these steps by running
1175 
1176  .. code-block:: none
1177 
1178  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1179 
1180  This will produce warped coaddTempExps for each visit. To coadd the
1181  warped data, we call ``assembleCoadd.py`` as follows:
1182 
1183  .. code-block:: none
1184 
1185  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1186  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1187  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1188  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1189  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1190  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1191  --selectId visit=903988 ccd=24
1192 
1193  This will process the HSC-I band data. The results are written in
1194  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1195 
1196  You may also choose to run:
1197 
1198  .. code-block:: none
1199 
1200  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1201  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1202  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1203  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1204  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1205  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1206  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1207  --selectId visit=903346 ccd=12
1208 
1209  to generate the coadd for the HSC-R band if you are interested in following
1210  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1211  """
1212  ConfigClass = SafeClipAssembleCoaddConfig
1213  _DefaultName = "safeClipAssembleCoadd"
1214 
1215  def __init__(self, *args, **kwargs):
1216  AssembleCoaddTask.__init__(self, *args, **kwargs)
1217  schema = afwTable.SourceTable.makeMinimalSchema()
1218  self.makeSubtask("clipDetection", schema=schema)
1219 
1220  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1221  """Assemble the coadd for a region.
1222 
1223  Compute the difference of coadds created with and without outlier
1224  rejection to identify coadd pixels that have outlier values in some
1225  individual visits.
1226  Detect clipped regions on the difference image and mark these regions
1227  on the one or two individual coaddTempExps where they occur if there
1228  is significant overlap between the clipped region and a source. This
1229  leaves us with a set of footprints from the difference image that have
1230  been identified as having occured on just one or two individual visits.
1231  However, these footprints were generated from a difference image. It
1232  is conceivable for a large diffuse source to have become broken up
1233  into multiple footprints acrosss the coadd difference in this process.
1234  Determine the clipped region from all overlapping footprints from the
1235  detected sources in each visit - these are big footprints.
1236  Combine the small and big clipped footprints and mark them on a new
1237  bad mask plane.
1238  Generate the coadd using `AssembleCoaddTask.run` without outlier
1239  removal. Clipped footprints will no longer make it into the coadd
1240  because they are marked in the new bad mask plane.
1241 
1242  Parameters
1243  ----------
1244  skyInfo : `lsst.pipe.base.Struct`
1245  Patch geometry information, from getSkyInfo
1246  tempExpRefList : `list`
1247  List of data reference to tempExp
1248  imageScalerList : `list`
1249  List of image scalers
1250  weightList : `list`
1251  List of weights
1252 
1253  Returns
1254  -------
1255  result : `lsst.pipe.base.Struct`
1256  Result struct with components:
1257 
1258  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1259  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
1260 
1261  Notes
1262  -----
1263  args and kwargs are passed but ignored in order to match the call
1264  signature expected by the parent task.
1265  """
1266  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1267  mask = exp.getMaskedImage().getMask()
1268  mask.addMaskPlane("CLIPPED")
1269 
1270  result = self.detectClip(exp, tempExpRefList)
1271 
1272  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1273 
1274  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1275  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1276  # Append big footprints from individual Warps to result.clipSpans
1277  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1278  result.detectionFootprints, maskClipValue, maskDetValue,
1279  exp.getBBox())
1280  # Create mask of the current clipped footprints
1281  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1282  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1283 
1284  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1285  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1286  maskClip |= maskClipBig
1287 
1288  # Assemble coadd from base class, but ignoring CLIPPED pixels
1289  badMaskPlanes = self.config.badMaskPlanes[:]
1290  badMaskPlanes.append("CLIPPED")
1291  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1292  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1293  result.clipSpans, mask=badPixelMask)
1294 
1295  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1296  """Return an exposure that contains the difference between unclipped
1297  and clipped coadds.
1298 
1299  Generate a difference image between clipped and unclipped coadds.
1300  Compute the difference image by subtracting an outlier-clipped coadd
1301  from an outlier-unclipped coadd. Return the difference image.
1302 
1303  Parameters
1304  ----------
1305  skyInfo : `lsst.pipe.base.Struct`
1306  Patch geometry information, from getSkyInfo
1307  tempExpRefList : `list`
1308  List of data reference to tempExp
1309  imageScalerList : `list`
1310  List of image scalers
1311  weightList : `list`
1312  List of weights
1313 
1314  Returns
1315  -------
1316  exp : `lsst.afw.image.Exposure`
1317  Difference image of unclipped and clipped coadd wrapped in an Exposure
1318  """
1319  # Clone and upcast self.config because current self.config is frozen
1320  config = AssembleCoaddConfig()
1321  # getattr necessary because subtasks do not survive Config.toDict()
1322  configIntersection = {k: getattr(self.config, k)
1323  for k, v in self.config.toDict().items() if (k in config.keys())}
1324  config.update(**configIntersection)
1325 
1326  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1327  config.statistic = 'MEAN'
1328  task = AssembleCoaddTask(config=config)
1329  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1330 
1331  config.statistic = 'MEANCLIP'
1332  task = AssembleCoaddTask(config=config)
1333  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1334 
1335  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1336  coaddDiff -= coaddClip.getMaskedImage()
1337  exp = afwImage.ExposureF(coaddDiff)
1338  exp.setPsf(coaddMean.getPsf())
1339  return exp
1340 
1341  def detectClip(self, exp, tempExpRefList):
1342  """Detect clipped regions on an exposure and set the mask on the
1343  individual tempExp masks.
1344 
1345  Detect footprints in the difference image after smoothing the
1346  difference image with a Gaussian kernal. Identify footprints that
1347  overlap with one or two input ``coaddTempExps`` by comparing the
1348  computed overlap fraction to thresholds set in the config. A different
1349  threshold is applied depending on the number of overlapping visits
1350  (restricted to one or two). If the overlap exceeds the thresholds,
1351  the footprint is considered "CLIPPED" and is marked as such on the
1352  coaddTempExp. Return a struct with the clipped footprints, the indices
1353  of the ``coaddTempExps`` that end up overlapping with the clipped
1354  footprints, and a list of new masks for the ``coaddTempExps``.
1355 
1356  Parameters
1357  ----------
1358  exp : `lsst.afw.image.Exposure`
1359  Exposure to run detection on.
1360  tempExpRefList : `list`
1361  List of data reference to tempExp.
1362 
1363  Returns
1364  -------
1365  result : `lsst.pipe.base.Struct`
1366  Result struct with components:
1367 
1368  - ``clipFootprints``: list of clipped footprints.
1369  - ``clipIndices``: indices for each ``clippedFootprint`` in
1370  ``tempExpRefList``.
1371  - ``clipSpans``: List of dictionaries containing spanSet lists
1372  to clip. Each element contains the new maskplane name
1373  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1374  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1375  compressed into footprints.
1376  """
1377  mask = exp.getMaskedImage().getMask()
1378  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1379  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1380  # Merge positive and negative together footprints together
1381  fpSet.positive.merge(fpSet.negative)
1382  footprints = fpSet.positive
1383  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1384  ignoreMask = self.getBadPixelMask()
1385 
1386  clipFootprints = []
1387  clipIndices = []
1388  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1389 
1390  # for use by detectClipBig
1391  visitDetectionFootprints = []
1392 
1393  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1394  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1395  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1396 
1397  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1398  for i, warpRef in enumerate(tempExpRefList):
1399  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1400  immediate=True).getMaskedImage().getMask()
1401  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1402  afwImage.PARENT, True)
1403  maskVisitDet &= maskDetValue
1404  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1405  visitDetectionFootprints.append(visitFootprints)
1406 
1407  for j, footprint in enumerate(footprints.getFootprints()):
1408  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1409  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1410 
1411  # build a list of clipped spans for each visit
1412  for j, footprint in enumerate(footprints.getFootprints()):
1413  nPixel = footprint.getArea()
1414  overlap = [] # hold the overlap with each visit
1415  indexList = [] # index of visit in global list
1416  for i in range(len(tempExpRefList)):
1417  ignore = ignoreArr[i, j]
1418  overlapDet = overlapDetArr[i, j]
1419  totPixel = nPixel - ignore
1420 
1421  # If we have more bad pixels than detection skip
1422  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1423  continue
1424  overlap.append(overlapDet/float(totPixel))
1425  indexList.append(i)
1426 
1427  overlap = numpy.array(overlap)
1428  if not len(overlap):
1429  continue
1430 
1431  keep = False # Should this footprint be marked as clipped?
1432  keepIndex = [] # Which tempExps does the clipped footprint belong to
1433 
1434  # If footprint only has one overlap use a lower threshold
1435  if len(overlap) == 1:
1436  if overlap[0] > self.config.minClipFootOverlapSingle:
1437  keep = True
1438  keepIndex = [0]
1439  else:
1440  # This is the general case where only visit should be clipped
1441  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1442  if len(clipIndex) == 1:
1443  keep = True
1444  keepIndex = [clipIndex[0]]
1445 
1446  # Test if there are clipped objects that overlap two different visits
1447  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1448  if len(clipIndex) == 2 and len(overlap) > 3:
1449  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1450  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1451  keep = True
1452  keepIndex = clipIndex
1453 
1454  if not keep:
1455  continue
1456 
1457  for index in keepIndex:
1458  globalIndex = indexList[index]
1459  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1460 
1461  clipIndices.append(numpy.array(indexList)[keepIndex])
1462  clipFootprints.append(footprint)
1463 
1464  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1465  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1466 
1467  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1468  maskClipValue, maskDetValue, coaddBBox):
1469  """Return individual warp footprints for large artifacts and append
1470  them to ``clipList`` in place.
1471 
1472  Identify big footprints composed of many sources in the coadd
1473  difference that may have originated in a large diffuse source in the
1474  coadd. We do this by indentifying all clipped footprints that overlap
1475  significantly with each source in all the coaddTempExps.
1476 
1477  Parameters
1478  ----------
1479  clipList : `list`
1480  List of alt mask SpanSets with clipping information. Modified.
1481  clipFootprints : `list`
1482  List of clipped footprints.
1483  clipIndices : `list`
1484  List of which entries in tempExpClipList each footprint belongs to.
1485  maskClipValue
1486  Mask value of clipped pixels.
1487  maskDetValue
1488  Mask value of detected pixels.
1489  coaddBBox : `lsst.afw.geom.Box`
1490  BBox of the coadd and warps.
1491 
1492  Returns
1493  -------
1494  bigFootprintsCoadd : `list`
1495  List of big footprints
1496  """
1497  bigFootprintsCoadd = []
1498  ignoreMask = self.getBadPixelMask()
1499  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1500  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1501  for footprint in visitFootprints.getFootprints():
1502  footprint.spans.setMask(maskVisitDet, maskDetValue)
1503 
1504  # build a mask of clipped footprints that are in this visit
1505  clippedFootprintsVisit = []
1506  for foot, clipIndex in zip(clipFootprints, clipIndices):
1507  if index not in clipIndex:
1508  continue
1509  clippedFootprintsVisit.append(foot)
1510  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1511  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1512 
1513  bigFootprintsVisit = []
1514  for foot in visitFootprints.getFootprints():
1515  if foot.getArea() < self.config.minBigOverlap:
1516  continue
1517  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1518  if nCount > self.config.minBigOverlap:
1519  bigFootprintsVisit.append(foot)
1520  bigFootprintsCoadd.append(foot)
1521 
1522  for footprint in bigFootprintsVisit:
1523  clippedSpans["CLIPPED"].append(footprint.spans)
1524 
1525  return bigFootprintsCoadd
1526 
1527 
1529  assembleStaticSkyModel = pexConfig.ConfigurableField(
1530  target=AssembleCoaddTask,
1531  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1532  " naive/first-iteration model of the static sky.",
1533  )
1534  detect = pexConfig.ConfigurableField(
1535  target=SourceDetectionTask,
1536  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1537  )
1538  detectTemplate = pexConfig.ConfigurableField(
1539  target=SourceDetectionTask,
1540  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1541  )
1542  maxNumEpochs = pexConfig.Field(
1543  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1544  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1545  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1546  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1547  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1548  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1549  "than transient and not masked.",
1550  dtype=int,
1551  default=2
1552  )
1553  maxFractionEpochsLow = pexConfig.RangeField(
1554  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1555  "Effective maxNumEpochs = "
1556  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1557  dtype=float,
1558  default=0.4,
1559  min=0., max=1.,
1560  )
1561  maxFractionEpochsHigh = pexConfig.RangeField(
1562  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1563  "Effective maxNumEpochs = "
1564  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1565  dtype=float,
1566  default=0.03,
1567  min=0., max=1.,
1568  )
1569  spatialThreshold = pexConfig.RangeField(
1570  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1571  "temporal criteria. If 0, clip all. If 1, clip none.",
1572  dtype=float,
1573  default=0.5,
1574  min=0., max=1.,
1575  inclusiveMin=True, inclusiveMax=True
1576  )
1577  doScaleWarpVariance = pexConfig.Field(
1578  doc="Rescale Warp variance plane using empirical noise?",
1579  dtype=bool,
1580  default=True,
1581  )
1582  scaleWarpVariance = pexConfig.ConfigurableField(
1583  target=ScaleVarianceTask,
1584  doc="Rescale variance on warps",
1585  )
1586  doPreserveContainedBySource = pexConfig.Field(
1587  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1588  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1589  dtype=bool,
1590  default=True,
1591  )
1592  doPrefilterArtifacts = pexConfig.Field(
1593  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1594  "because they will be excluded anyway. This prevents them from contributing "
1595  "to the outlier epoch count image and potentially being labeled as persistant."
1596  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1597  dtype=bool,
1598  default=True
1599  )
1600  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1601  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1602  dtype=str,
1603  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1604  )
1605  prefilterArtifactsRatio = pexConfig.Field(
1606  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1607  dtype=float,
1608  default=0.05
1609  )
1610 
1611  def setDefaults(self):
1612  AssembleCoaddConfig.setDefaults(self)
1613  self.statistic = 'MEAN'
1615 
1616  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1617  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1618  if "EDGE" in self.badMaskPlanes:
1619  self.badMaskPlanes.remove('EDGE')
1620  self.removeMaskPlanes.append('EDGE')
1621  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1622  self.assembleStaticSkyModel.warpType = 'psfMatched'
1623  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1624  self.assembleStaticSkyModel.sigmaClip = 2.5
1625  self.assembleStaticSkyModel.clipIter = 3
1626  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1627  self.assembleStaticSkyModel.doWrite = False
1628  self.detect.doTempLocalBackground = False
1629  self.detect.reEstimateBackground = False
1630  self.detect.returnOriginalFootprints = False
1631  self.detect.thresholdPolarity = "both"
1632  self.detect.thresholdValue = 5
1633  self.detect.nSigmaToGrow = 2
1634  self.detect.minPixels = 4
1635  self.detect.isotropicGrow = True
1636  self.detect.thresholdType = "pixel_stdev"
1637  self.detectTemplate.nSigmaToGrow = 2.0
1638  self.detectTemplate.doTempLocalBackground = False
1639  self.detectTemplate.reEstimateBackground = False
1640  self.detectTemplate.returnOriginalFootprints = False
1641 
1642 
1644  """Assemble a compareWarp coadded image from a set of warps
1645  by masking artifacts detected by comparing PSF-matched warps.
1646 
1647  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1648  we clip outliers). The problem with doing this is that when computing the
1649  coadd PSF at a given location, individual visit PSFs from visits with
1650  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1651  In this task, we correct for this behavior by creating a new badMaskPlane
1652  'CLIPPED' which marks pixels in the individual warps suspected to contain
1653  an artifact. We populate this plane on the input warps by comparing
1654  PSF-matched warps with a PSF-matched median coadd which serves as a
1655  model of the static sky. Any group of pixels that deviates from the
1656  PSF-matched template coadd by more than config.detect.threshold sigma,
1657  is an artifact candidate. The candidates are then filtered to remove
1658  variable sources and sources that are difficult to subtract such as
1659  bright stars. This filter is configured using the config parameters
1660  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1661  the maximum fraction of epochs that the deviation can appear in and still
1662  be considered an artifact. The spatialThreshold is the maximum fraction of
1663  pixels in the footprint of the deviation that appear in other epochs
1664  (where other epochs is defined by the temporalThreshold). If the deviant
1665  region meets this criteria of having a significant percentage of pixels
1666  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1667  set in the mask. These regions will not contribute to the final coadd.
1668  Furthermore, any routine to determine the coadd PSF can now be cognizant
1669  of clipped regions. Note that the algorithm implemented by this task is
1670  preliminary and works correctly for HSC data. Parameter modifications and
1671  or considerable redesigning of the algorithm is likley required for other
1672  surveys.
1673 
1674  ``CompareWarpAssembleCoaddTask`` sub-classes
1675  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1676  as a subtask to generate the TemplateCoadd (the model of the static sky).
1677 
1678  Notes
1679  -----
1680  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1681  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1682  ``baseDebug`` for more about ``debug.py`` files.
1683 
1684  This task supports the following debug variables:
1685 
1686  - ``saveCountIm``
1687  If True then save the Epoch Count Image as a fits file in the `figPath`
1688  - ``figPath``
1689  Path to save the debug fits images and figures
1690 
1691  For example, put something like:
1692 
1693  .. code-block:: python
1694 
1695  import lsstDebug
1696  def DebugInfo(name):
1697  di = lsstDebug.getInfo(name)
1698  if name == "lsst.pipe.tasks.assembleCoadd":
1699  di.saveCountIm = True
1700  di.figPath = "/desired/path/to/debugging/output/images"
1701  return di
1702  lsstDebug.Info = DebugInfo
1703 
1704  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1705  ``--debug`` flag. Some subtasks may have their own debug variables;
1706  see individual Task documentation.
1707 
1708  Examples
1709  --------
1710  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1711  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1712  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1713  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1714  and filter to be coadded (specified using
1715  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1716  along with a list of coaddTempExps to attempt to coadd (specified using
1717  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1718  Only the warps that cover the specified tract and patch will be coadded.
1719  A list of the available optional arguments can be obtained by calling
1720  ``assembleCoadd.py`` with the ``--help`` command line argument:
1721 
1722  .. code-block:: none
1723 
1724  assembleCoadd.py --help
1725 
1726  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1727  context of multi-band processing, we will generate the HSC-I & -R band
1728  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1729  To begin, assuming that the lsst stack has been already set up, we must
1730  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1731  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1732  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1733  directory. To begin assembling the coadds, we must first
1734 
1735  - processCcd
1736  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1737  - makeSkyMap
1738  create a skymap that covers the area of the sky present in the raw exposures
1739  - makeCoaddTempExp
1740  warp the individual calibrated exposures to the tangent plane of the coadd
1741 
1742  We can perform all of these steps by running
1743 
1744  .. code-block:: none
1745 
1746  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1747 
1748  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1749  warped data, we call ``assembleCoadd.py`` as follows:
1750 
1751  .. code-block:: none
1752 
1753  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1754  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1755  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1756  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1757  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1758  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1759  --selectId visit=903988 ccd=24
1760 
1761  This will process the HSC-I band data. The results are written in
1762  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1763  """
1764  ConfigClass = CompareWarpAssembleCoaddConfig
1765  _DefaultName = "compareWarpAssembleCoadd"
1766 
1767  def __init__(self, *args, **kwargs):
1768  AssembleCoaddTask.__init__(self, *args, **kwargs)
1769  self.makeSubtask("assembleStaticSkyModel")
1770  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1771  self.makeSubtask("detect", schema=detectionSchema)
1772  if self.config.doPreserveContainedBySource:
1773  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1774  if self.config.doScaleWarpVariance:
1775  self.makeSubtask("scaleWarpVariance")
1776 
1777  def makeSupplementaryData(self, dataRef, selectDataList):
1778  """Make inputs specific to Subclass.
1779 
1780  Generate a templateCoadd to use as a native model of static sky to
1781  subtract from warps.
1782 
1783  Parameters
1784  ----------
1785  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1786  Butler dataRef for supplementary data.
1787  selectDataList : `list`
1788  List of data references to Warps.
1789 
1790  Returns
1791  -------
1792  result : `lsst.pipe.base.Struct`
1793  Result struct with components:
1794 
1795  - ``templateCoaddcoadd``: coadded exposure (``lsst.afw.image.Exposure``).
1796  """
1797  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList)
1798 
1799  if templateCoadd is None:
1800  warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1801  self.assembleStaticSkyModel.warpType[1:])
1802  message = """No %(warpName)s warps were found to build the template coadd which is
1803  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1804  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1805  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1806 
1807  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1808  another algorithm like:
1809 
1810  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1811  config.assemble.retarget(SafeClipAssembleCoaddTask)
1812  """ % {"warpName": warpName}
1813  raise RuntimeError(message)
1814 
1815  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
1816  nImage=templateCoadd.nImage)
1817 
1818  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1819  supplementaryData, *args, **kwargs):
1820  """Assemble the coadd.
1821 
1822  Find artifacts and apply them to the warps' masks creating a list of
1823  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
1824  plane. Then pass these alternative masks to the base class's `run`
1825  method.
1826 
1827  Parameters
1828  ----------
1829  skyInfo : `lsst.pipe.base.Struct`
1830  Patch geometry information.
1831  tempExpRefList : `list`
1832  List of data references to warps.
1833  imageScalerList : `list`
1834  List of image scalers.
1835  weightList : `list`
1836  List of weights.
1837  supplementaryData : `lsst.pipe.base.Struct`
1838  This Struct must contain a ``templateCoadd`` that serves as the
1839  model of the static sky.
1840 
1841  Returns
1842  -------
1843  result : `lsst.pipe.base.Struct`
1844  Result struct with components:
1845 
1846  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1847  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
1848  """
1849  templateCoadd = supplementaryData.templateCoadd
1850  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1851  badMaskPlanes = self.config.badMaskPlanes[:]
1852  badMaskPlanes.append("CLIPPED")
1853  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1854 
1855  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1856  spanSetMaskList, mask=badPixelMask)
1857 
1858  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
1859  # Psf-Matching moves the real edge inwards
1860  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1861  return result
1862 
1863  def applyAltEdgeMask(self, mask, altMaskList):
1864  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1865 
1866  Parameters
1867  ----------
1868  mask : `lsst.afw.image.Mask`
1869  Original mask.
1870  altMaskList : `list`
1871  List of Dicts containing ``spanSet`` lists.
1872  Each element contains the new mask plane name (e.g. "CLIPPED
1873  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
1874  the mask.
1875  """
1876  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
1877  for visitMask in altMaskList:
1878  if "EDGE" in visitMask:
1879  for spanSet in visitMask['EDGE']:
1880  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1881 
1882  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1883  """Find artifacts.
1884 
1885  Loop through warps twice. The first loop builds a map with the count
1886  of how many epochs each pixel deviates from the templateCoadd by more
1887  than ``config.chiThreshold`` sigma. The second loop takes each
1888  difference image and filters the artifacts detected in each using
1889  count map to filter out variable sources and sources that are
1890  difficult to subtract cleanly.
1891 
1892  Parameters
1893  ----------
1894  templateCoadd : `lsst.afw.image.Exposure`
1895  Exposure to serve as model of static sky.
1896  tempExpRefList : `list`
1897  List of data references to warps.
1898  imageScalerList : `list`
1899  List of image scalers.
1900 
1901  Returns
1902  -------
1903  altMasks : `list`
1904  List of dicts containing information about CLIPPED
1905  (i.e., artifacts), NO_DATA, and EDGE pixels.
1906  """
1907 
1908  self.log.debug("Generating Count Image, and mask lists.")
1909  coaddBBox = templateCoadd.getBBox()
1910  slateIm = afwImage.ImageU(coaddBBox)
1911  epochCountImage = afwImage.ImageU(coaddBBox)
1912  nImage = afwImage.ImageU(coaddBBox)
1913  spanSetArtifactList = []
1914  spanSetNoDataMaskList = []
1915  spanSetEdgeList = []
1916  badPixelMask = self.getBadPixelMask()
1917 
1918  # mask of the warp diffs should = that of only the warp
1919  templateCoadd.mask.clearAllMaskPlanes()
1920 
1921  if self.config.doPreserveContainedBySource:
1922  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1923  else:
1924  templateFootprints = None
1925 
1926  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1927  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1928  if warpDiffExp is not None:
1929  # This nImage only approximates the final nImage because it uses the PSF-matched mask
1930  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1931  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1932  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1933  fpSet.positive.merge(fpSet.negative)
1934  footprints = fpSet.positive
1935  slateIm.set(0)
1936  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1937 
1938  # Remove artifacts due to defects before they contribute to the epochCountImage
1939  if self.config.doPrefilterArtifacts:
1940  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
1941  for spans in spanSetList:
1942  spans.setImage(slateIm, 1, doClip=True)
1943  epochCountImage += slateIm
1944 
1945  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1946  # undergo a second convolution. Pixels with data in the direct warp
1947  # but not in the PSF-matched warp will not have their artifacts detected.
1948  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1949  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1950  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1951  nansMask.setXY0(warpDiffExp.getXY0())
1952  edgeMask = warpDiffExp.mask
1953  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1954  edgeMask.getPlaneBitMask("EDGE")).split()
1955  else:
1956  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1957  # In this case, mask the whole epoch
1958  nansMask = afwImage.MaskX(coaddBBox, 1)
1959  spanSetList = []
1960  spanSetEdgeMask = []
1961 
1962  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1963 
1964  spanSetNoDataMaskList.append(spanSetNoDataMask)
1965  spanSetArtifactList.append(spanSetList)
1966  spanSetEdgeList.append(spanSetEdgeMask)
1967 
1968  if lsstDebug.Info(__name__).saveCountIm:
1969  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1970  epochCountImage.writeFits(path)
1971 
1972  for i, spanSetList in enumerate(spanSetArtifactList):
1973  if spanSetList:
1974  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
1975  templateFootprints)
1976  spanSetArtifactList[i] = filteredSpanSetList
1977 
1978  altMasks = []
1979  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1980  altMasks.append({'CLIPPED': artifacts,
1981  'NO_DATA': noData,
1982  'EDGE': edge})
1983  return altMasks
1984 
1985  def prefilterArtifacts(self, spanSetList, exp):
1986  """Remove artifact candidates covered by bad mask plane.
1987 
1988  Any future editing of the candidate list that does not depend on
1989  temporal information should go in this method.
1990 
1991  Parameters
1992  ----------
1993  spanSetList : `list`
1994  List of SpanSets representing artifact candidates.
1995  exp : `lsst.afw.image.Exposure`
1996  Exposure containing mask planes used to prefilter.
1997 
1998  Returns
1999  -------
2000  returnSpanSetList : `list`
2001  List of SpanSets with artifacts.
2002  """
2003  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2004  goodArr = (exp.mask.array & badPixelMask) == 0
2005  returnSpanSetList = []
2006  bbox = exp.getBBox()
2007  x0, y0 = exp.getXY0()
2008  for i, span in enumerate(spanSetList):
2009  y, x = span.clippedTo(bbox).indices()
2010  yIndexLocal = numpy.array(y) - y0
2011  xIndexLocal = numpy.array(x) - x0
2012  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2013  if goodRatio > self.config.prefilterArtifactsRatio:
2014  returnSpanSetList.append(span)
2015  return returnSpanSetList
2016 
2017  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2018  """Filter artifact candidates.
2019 
2020  Parameters
2021  ----------
2022  spanSetList : `list`
2023  List of SpanSets representing artifact candidates.
2024  epochCountImage : `lsst.afw.image.Image`
2025  Image of accumulated number of warpDiff detections.
2026  nImage : `lsst.afw.image.Image`
2027  Image of the accumulated number of total epochs contributing.
2028 
2029  Returns
2030  -------
2031  maskSpanSetList : `list`
2032  List of SpanSets with artifacts.
2033  """
2034 
2035  maskSpanSetList = []
2036  x0, y0 = epochCountImage.getXY0()
2037  for i, span in enumerate(spanSetList):
2038  y, x = span.indices()
2039  yIdxLocal = [y1 - y0 for y1 in y]
2040  xIdxLocal = [x1 - x0 for x1 in x]
2041  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2042  totalN = nImage.array[yIdxLocal, xIdxLocal]
2043 
2044  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2045  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2046  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2047  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2048  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2049  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2050  (outlierN <= effectiveMaxNumEpochs))
2051  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2052  if percentBelowThreshold > self.config.spatialThreshold:
2053  maskSpanSetList.append(span)
2054 
2055  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2056  # If a candidate is contained by a footprint on the template coadd, do not clip
2057  filteredMaskSpanSetList = []
2058  for span in maskSpanSetList:
2059  doKeep = True
2060  for footprint in footprintsToExclude.positive.getFootprints():
2061  if footprint.spans.contains(span):
2062  doKeep = False
2063  break
2064  if doKeep:
2065  filteredMaskSpanSetList.append(span)
2066  maskSpanSetList = filteredMaskSpanSetList
2067 
2068  return maskSpanSetList
2069 
2070  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2071  """Fetch a warp from the butler and return a warpDiff.
2072 
2073  Parameters
2074  ----------
2075  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2076  Butler dataRef for the warp.
2077  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2078  An image scaler object.
2079  templateCoadd : `lsst.afw.image.Exposure`
2080  Exposure to be substracted from the scaled warp.
2081 
2082  Returns
2083  -------
2084  warp : `lsst.afw.image.Exposure`
2085  Exposure of the image difference between the warp and template.
2086  """
2087 
2088  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2089  warpName = self.getTempExpDatasetName('psfMatched')
2090  if not warpRef.datasetExists(warpName):
2091  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2092  return None
2093  warp = warpRef.get(warpName, immediate=True)
2094  # direct image scaler OK for PSF-matched Warp
2095  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2096  mi = warp.getMaskedImage()
2097  if self.config.doScaleWarpVariance:
2098  try:
2099  self.scaleWarpVariance.run(mi)
2100  except Exception as exc:
2101  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2102  mi -= templateCoadd.getMaskedImage()
2103  return warp
2104 
2105  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2106  """Return a path to which to write debugging output.
2107 
2108  Creates a hyphen-delimited string of dataId values for simple filenames.
2109 
2110  Parameters
2111  ----------
2112  prefix : `str`
2113  Prefix for filename.
2114  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2115  Butler dataRef to make the path from.
2116  coaddLevel : `bool`, optional.
2117  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2118  'filter', but no 'visit').
2119 
2120  Returns
2121  -------
2122  result : `str`
2123  Path for debugging output.
2124  """
2125  if coaddLevel:
2126  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2127  else:
2128  keys = warpRef.dataId.keys()
2129  keyList = sorted(keys, reverse=True)
2130  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2131  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2132  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:171
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:94
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def makeSupplementaryData(self, dataRef, selectDataList)
def getTempExpRefList(self, patchRef, calExpRefList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:127
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def runDataRef(self, dataRef, selectDataList=[])
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:221
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:107
def processResults(self, coaddExposure, dataRef)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)