lsst.pipe.tasks  17.0.1-5-gf0ac6446+11
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy
24 import warnings
25 import lsst.pex.config as pexConfig
26 import lsst.pex.exceptions as pexExceptions
27 import lsst.afw.geom as afwGeom
28 import lsst.afw.image as afwImage
29 import lsst.afw.math as afwMath
30 import lsst.afw.table as afwTable
31 import lsst.afw.detection as afwDet
32 import lsst.coadd.utils as coaddUtils
33 import lsst.pipe.base as pipeBase
34 import lsst.meas.algorithms as measAlg
35 import lsst.log as log
36 import lsstDebug
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo
38 from .interpImage import InterpImageTask
39 from .scaleZeroPoint import ScaleZeroPointTask
40 from .coaddHelpers import groupPatchExposures, getGroupDataRef
41 from .scaleVariance import ScaleVarianceTask
42 from lsst.meas.algorithms import SourceDetectionTask
43 from lsst.pipe.base.shims import ShimButler
44 
45 __all__ = ["AssembleCoaddTask", "AssembleCoaddConfig", "SafeClipAssembleCoaddTask",
46  "SafeClipAssembleCoaddConfig", "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
47 
48 
49 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig):
50  """Configuration parameters for the `AssembleCoaddTask`.
51 
52  Notes
53  -----
54  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
55  only set the bitplane config.brightObjectMaskName. To make this useful you
56  *must* also configure the flags.pixel algorithm, for example by adding
57 
58  .. code-block:: none
59 
60  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
61  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
62 
63  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
64  """
65  warpType = pexConfig.Field(
66  doc="Warp name: one of 'direct' or 'psfMatched'",
67  dtype=str,
68  default="direct",
69  )
70  subregionSize = pexConfig.ListField(
71  dtype=int,
72  doc="Width, height of stack subregion size; "
73  "make small enough that a full stack of images will fit into memory at once.",
74  length=2,
75  default=(2000, 2000),
76  )
77  statistic = pexConfig.Field(
78  dtype=str,
79  doc="Main stacking statistic for aggregating over the epochs.",
80  default="MEANCLIP",
81  )
82  doSigmaClip = pexConfig.Field(
83  dtype=bool,
84  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
85  default=False,
86  )
87  sigmaClip = pexConfig.Field(
88  dtype=float,
89  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
90  default=3.0,
91  )
92  clipIter = pexConfig.Field(
93  dtype=int,
94  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
95  default=2,
96  )
97  calcErrorFromInputVariance = pexConfig.Field(
98  dtype=bool,
99  doc="Calculate coadd variance from input variance by stacking statistic."
100  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
101  default=True,
102  )
103  scaleZeroPoint = pexConfig.ConfigurableField(
104  target=ScaleZeroPointTask,
105  doc="Task to adjust the photometric zero point of the coadd temp exposures",
106  )
107  doInterp = pexConfig.Field(
108  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
109  dtype=bool,
110  default=True,
111  )
112  interpImage = pexConfig.ConfigurableField(
113  target=InterpImageTask,
114  doc="Task to interpolate (and extrapolate) over NaN pixels",
115  )
116  doWrite = pexConfig.Field(
117  doc="Persist coadd?",
118  dtype=bool,
119  default=True,
120  )
121  doNImage = pexConfig.Field(
122  doc="Create image of number of contributing exposures for each pixel",
123  dtype=bool,
124  default=False,
125  )
126  doUsePsfMatchedPolygons = pexConfig.Field(
127  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
128  dtype=bool,
129  default=False,
130  )
131  maskPropagationThresholds = pexConfig.DictField(
132  keytype=str,
133  itemtype=float,
134  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
135  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
136  "would have contributed exceeds this value."),
137  default={"SAT": 0.1},
138  )
139  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
140  doc="Mask planes to remove before coadding")
141  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
142  doc="Set mask and flag bits for bright objects?")
143  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
144  doc="Name of mask bit used for bright objects")
145  coaddPsf = pexConfig.ConfigField(
146  doc="Configuration for CoaddPsf",
147  dtype=measAlg.CoaddPsfConfig,
148  )
149  doAttachTransmissionCurve = pexConfig.Field(
150  dtype=bool, default=False, optional=False,
151  doc=("Attach a piecewise TransmissionCurve for the coadd? "
152  "(requires all input Exposures to have TransmissionCurves).")
153  )
154  inputWarps = pipeBase.InputDatasetField(
155  doc=("Input list of warps to be assemebled i.e. stacked."
156  "WarpType (e.g. direct, psfMatched) is controlled by we warpType config parameter"),
157  nameTemplate="{inputCoaddName}Coadd_{warpType}Warp",
158  storageClass="ExposureF",
159  dimensions=("Tract", "Patch", "SkyMap", "Visit", "Instrument"),
160  manualLoad=True,
161  )
162  skyMap = pipeBase.InputDatasetField(
163  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
164  nameTemplate="{inputCoaddName}Coadd_skyMap",
165  storageClass="SkyMap",
166  dimensions=("SkyMap", ),
167  scalar=True
168  )
169  brightObjectMask = pipeBase.InputDatasetField(
170  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
171  " BRIGHT_OBJECT."),
172  name="brightObjectMask",
173  storageClass="ObjectMaskCatalog",
174  dimensions=("Tract", "Patch", "SkyMap", "AbstractFilter"),
175  scalar=True
176  )
177  coaddExposure = pipeBase.OutputDatasetField(
178  doc="Output coadded exposure, produced by stacking input warps",
179  nameTemplate="{outputCoaddName}Coadd",
180  storageClass="ExposureF",
181  dimensions=("Tract", "Patch", "SkyMap", "AbstractFilter"),
182  scalar=True
183  )
184  nImage = pipeBase.OutputDatasetField(
185  doc="Output image of number of input images per pixel",
186  nameTemplate="{outputCoaddName}Coadd_nImage",
187  storageClass="ImageU",
188  dimensions=("Tract", "Patch", "SkyMap", "AbstractFilter"),
189  scalar=True
190  )
191 
192  def setDefaults(self):
193  super().setDefaults()
194  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
195  self.formatTemplateNames({"inputCoaddName": "deep", "outputCoaddName": "deep",
196  "warpType": self.warpType})
197  self.quantum.dimensions = ("Tract", "Patch", "AbstractFilter", "SkyMap")
198 
199  def validate(self):
200  super().validate()
201  if self.doPsfMatch:
202  # Backwards compatibility.
203  # Configs do not have loggers
204  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
205  self.warpType = 'psfMatched'
206  if self.doSigmaClip and self.statistic != "MEANCLIP":
207  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
208  self.statistic = "MEANCLIP"
209  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
210  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
211  "compute and set a non-zero coadd variance estimate." % (self.statistic))
212 
213  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
214  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
215  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
216  if str(k) not in unstackableStats]
217  raise ValueError("statistic %s is not allowed. Please choose one of %s."
218  % (self.statistic, stackableStats))
219 
220 
221 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
222  """Assemble a coadded image from a set of warps (coadded temporary exposures).
223 
224  We want to assemble a coadded image from a set of Warps (also called
225  coadded temporary exposures or ``coaddTempExps``).
226  Each input Warp covers a patch on the sky and corresponds to a single
227  run/visit/exposure of the covered patch. We provide the task with a list
228  of Warps (``selectDataList``) from which it selects Warps that cover the
229  specified patch (pointed at by ``dataRef``).
230  Each Warp that goes into a coadd will typically have an independent
231  photometric zero-point. Therefore, we must scale each Warp to set it to
232  a common photometric zeropoint. WarpType may be one of 'direct' or
233  'psfMatched', and the boolean configs `config.makeDirect` and
234  `config.makePsfMatched` set which of the warp types will be coadded.
235  The coadd is computed as a mean with optional outlier rejection.
236  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
237  Finally, Warps can have bad 'NaN' pixels which received no input from the
238  source calExps. We interpolate over these bad (NaN) pixels.
239 
240  `AssembleCoaddTask` uses several sub-tasks. These are
241 
242  - `ScaleZeroPointTask`
243  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
244  - `InterpImageTask`
245  - interpolate across bad pixels (NaN) in the final coadd
246 
247  You can retarget these subtasks if you wish.
248 
249  Notes
250  -----
251  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
252  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
253  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
254  no debug variables of its own. Some of the subtasks may support debug
255  variables. See the documentation for the subtasks for further information.
256 
257  Examples
258  --------
259  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
260  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
261  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
262  inputs: a data reference to the tract patch and filter to be coadded, and
263  a list of Warps to attempt to coadd. These are specified using ``--id`` and
264  ``--selectId``, respectively:
265 
266  .. code-block:: none
267 
268  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
269  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
270 
271  Only the Warps that cover the specified tract and patch will be coadded.
272  A list of the available optional arguments can be obtained by calling
273  ``assembleCoadd.py`` with the ``--help`` command line argument:
274 
275  .. code-block:: none
276 
277  assembleCoadd.py --help
278 
279  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
280  multi-band processing, we will generate the HSC-I & -R band coadds from
281  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
282  assuming that the lsst stack has been already set up, we must set up the
283  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
284  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
285  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
286  coadds, we must first
287 
288  - processCcd
289  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
290  - makeSkyMap
291  - create a skymap that covers the area of the sky present in the raw exposures
292  - makeCoaddTempExp
293  - warp the individual calibrated exposures to the tangent plane of the coadd
294 
295  We can perform all of these steps by running
296 
297  .. code-block:: none
298 
299  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
300 
301  This will produce warped exposures for each visit. To coadd the warped
302  data, we call assembleCoadd.py as follows:
303 
304  .. code-block:: none
305 
306  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
307  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
308  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
309  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
310  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
311  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
312  --selectId visit=903988 ccd=24
313 
314  that will process the HSC-I band data. The results are written in
315  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
316 
317  You may also choose to run:
318 
319  .. code-block:: none
320 
321  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
322  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
323  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
324  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
325  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
326  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
327  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
328  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
329 
330  to generate the coadd for the HSC-R band if you are interested in
331  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
332  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
333  rather than `AssembleCoaddTask` to make the coadd.
334  """
335  ConfigClass = AssembleCoaddConfig
336  _DefaultName = "assembleCoadd"
337 
338  def __init__(self, *args, **kwargs):
339  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
340  if args:
341  argNames = ["config", "name", "parentTask", "log"]
342  kwargs.update({k: v for k, v in zip(argNames, args)})
343  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
344  "PipelineTask will not take positional args" % argNames, FutureWarning)
345 
346  super().__init__(**kwargs)
347  self.makeSubtask("interpImage")
348  self.makeSubtask("scaleZeroPoint")
349 
350  if self.config.doMaskBrightObjects:
351  mask = afwImage.Mask()
352  try:
353  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
354  except pexExceptions.LsstCppException:
355  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
356  mask.getMaskPlaneDict().keys())
357  del mask
358 
359  self.warpType = self.config.warpType
360 
361  @classmethod
362  def getOutputDatasetTypes(cls, config):
363  """Return output dataset type descriptors
364 
365  Remove output dataset types not produced by the Task
366  """
367  outputTypeDict = super().getOutputDatasetTypes(config)
368  if not config.doNImage:
369  outputTypeDict.pop("nImage", None)
370  return outputTypeDict
371 
372  @classmethod
373  def getInputDatasetTypes(cls, config):
374  """Return input dataset type descriptors
375 
376  Remove input dataset types not used by the Task
377  """
378  inputTypeDict = super().getInputDatasetTypes(config)
379  if not config.doMaskBrightObjects:
380  inputTypeDict.pop("brightObjectMask", None)
381  return inputTypeDict
382 
383  @classmethod
384  def getPrerequisiteDatasetTypes(cls, config):
385  return frozenset(["brightObjectMask"])
386 
387  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
388  """Assemble a coadd from a set of Warps.
389 
390  PipelineTask (Gen3) entry point to Coadd a set of Warps.
391  Analogous to `runDataRef`, it prepares all the data products to be
392  passed to `run`, and processes the results before returning to struct
393  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
394  Therefore, its inputs are accessed subregion by subregion
395  by the `lsst.daf.butler.ShimButler` that quacks like a Gen2
396  `lsst.daf.persistence.Butler`. Updates to this method should
397  correspond to an update in `runDataRef` while both entry points
398  are used.
399 
400  Parameters
401  ----------
402  inputData : `dict`
403  Keys are the names of the configs describing input dataset types.
404  Values are input Python-domain data objects (or lists of objects)
405  retrieved from data butler.
406  inputDataIds : `dict`
407  Keys are the names of the configs describing input dataset types.
408  Values are DataIds (or lists of DataIds) that task consumes for
409  corresponding dataset type.
410  outputDataIds : `dict`
411  Keys are the names of the configs describing input dataset types.
412  Values are DataIds (or lists of DataIds) that task is to produce
413  for corresponding dataset type.
414  butler : `lsst.daf.butler.Butler`
415  Gen3 Butler object for fetching additional data products before
416  running the Task
417 
418  Returns
419  -------
420  result : `lsst.pipe.base.Struct`
421  Result struct with components:
422 
423  - ``coaddExposure`` : coadded exposure (``lsst.afw.image.Exposure``)
424  - ``nImage``: N Image (``lsst.afw.image.Image``)
425  """
426  # Construct skyInfo expected by run
427  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
428  skyMap = inputData["skyMap"]
429  outputDataId = next(iter(outputDataIds.values()))
430  inputData['skyInfo'] = makeSkyInfo(skyMap,
431  tractId=outputDataId['tract'],
432  patchId=outputDataId['patch'])
433 
434  # Construct list of input Shim DataRefs that quack like Gen2 DataRefs
435  butlerShim = ShimButler(butler)
436  warpRefList = [butlerShim.dataRef(self.config.inputWarps.name, dataId=dataId)
437  for dataId in inputDataIds['inputWarps']]
438 
439  # Construct output Shim DataRef
440  patchRef = butlerShim.dataRef(self.config.coaddExposure.name, dataId=outputDataIds['coaddExposure'])
441 
442  # Perform same middle steps as `runDataRef` does
443  inputs = self.prepareInputs(warpRefList)
444  self.log.info("Found %d %s", len(inputs.tempExpRefList),
445  self.getTempExpDatasetName(self.warpType))
446  if len(inputs.tempExpRefList) == 0:
447  self.log.warn("No coadd temporary exposures found")
448  return
449 
450  supplementaryData = self.makeSupplementaryDataGen3(inputData, inputDataIds, outputDataIds, butler)
451 
452  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
453  inputs.weightList, supplementaryData=supplementaryData)
454 
455  self.processResults(retStruct.coaddExposure, patchRef)
456  return retStruct
457 
458  @pipeBase.timeMethod
459  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
460  """Assemble a coadd from a set of Warps.
461 
462  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
463  Compute weights to be applied to each Warp and
464  find scalings to match the photometric zeropoint to a reference Warp.
465  Assemble the Warps using `run`. Interpolate over NaNs and
466  optionally write the coadd to disk. Return the coadded exposure.
467 
468  Parameters
469  ----------
470  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
471  Data reference defining the patch for coaddition and the
472  reference Warp (if ``config.autoReference=False``).
473  Used to access the following data products:
474  - ``self.config.coaddName + "Coadd_skyMap"``
475  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
476  - ``self.config.coaddName + "Coadd"``
477  selectDataList : `list`
478  List of data references to Calexps. Data to be coadded will be
479  selected from this list based on overlap with the patch defined
480  by dataRef, grouped by visit, and converted to a list of data
481  references to warps.
482  warpRefList : `list`
483  List of data references to Warps to be coadded.
484  Note: `warpRefList` is just the new name for `tempExpRefList`.
485 
486  Returns
487  -------
488  retStruct : `lsst.pipe.base.Struct`
489  Result struct with components:
490 
491  - ``coaddExposure``: coadded exposure (``Exposure``).
492  - ``nImage``: exposure count image (``Image``).
493  """
494  if selectDataList and warpRefList:
495  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
496  "and which to use is ambiguous. Please pass only one.")
497 
498  skyInfo = self.getSkyInfo(dataRef)
499  if warpRefList is None:
500  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
501  if len(calExpRefList) == 0:
502  self.log.warn("No exposures to coadd")
503  return
504  self.log.info("Coadding %d exposures", len(calExpRefList))
505 
506  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
507 
508  inputData = self.prepareInputs(warpRefList)
509  self.log.info("Found %d %s", len(inputData.tempExpRefList),
510  self.getTempExpDatasetName(self.warpType))
511  if len(inputData.tempExpRefList) == 0:
512  self.log.warn("No coadd temporary exposures found")
513  return
514 
515  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
516 
517  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
518  inputData.weightList, supplementaryData=supplementaryData)
519 
520  self.processResults(retStruct.coaddExposure, dataRef)
521  if self.config.doWrite:
522  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
523  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
524  if self.config.doNImage and retStruct.nImage is not None:
525  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
526 
527  return retStruct
528 
529  def processResults(self, coaddExposure, dataRef):
530  """Interpolate over missing data and mask bright stars.
531 
532  Parameters
533  ----------
534  coaddExposure : `lsst.afw.image.Exposure`
535  The coadded exposure to process.
536  dataRef : `lsst.daf.persistence.ButlerDataRef`
537  Butler data reference for supplementary data.
538  """
539  if self.config.doInterp:
540  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
541  # The variance must be positive; work around for DM-3201.
542  varArray = coaddExposure.variance.array
543  with numpy.errstate(invalid="ignore"):
544  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
545 
546  if self.config.doMaskBrightObjects:
547  brightObjectMasks = self.readBrightObjectMasks(dataRef)
548  self.setBrightObjectMasks(coaddExposure, dataRef.dataId, brightObjectMasks)
549 
550  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
551  """Make additional inputs to run() specific to subclasses (Gen2)
552 
553  Duplicates interface of `runDataRef` method
554  Available to be implemented by subclasses only if they need the
555  coadd dataRef for performing preliminary processing before
556  assembling the coadd.
557 
558  Parameters
559  ----------
560  dataRef : `lsst.daf.persistence.ButlerDataRef`
561  Butler data reference for supplementary data.
562  selectDataList : `list`
563  List of data references to Warps.
564  """
565  pass
566 
567  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
568  """Make additional inputs to run() specific to subclasses (Gen3)
569 
570  Duplicates interface of`adaptArgsAndRun` method.
571  Available to be implemented by subclasses only if they need the
572  coadd dataRef for performing preliminary processing before
573  assembling the coadd.
574 
575  Parameters
576  ----------
577  inputData : `dict`
578  Keys are the names of the configs describing input dataset types.
579  Values are input Python-domain data objects (or lists of objects)
580  retrieved from data butler.
581  inputDataIds : `dict`
582  Keys are the names of the configs describing input dataset types.
583  Values are DataIds (or lists of DataIds) that task consumes for
584  corresponding dataset type.
585  DataIds are guaranteed to match data objects in ``inputData``.
586  outputDataIds : `dict`
587  Keys are the names of the configs describing input dataset types.
588  Values are DataIds (or lists of DataIds) that task is to produce
589  for corresponding dataset type.
590  butler : `lsst.daf.butler.Butler`
591  Gen3 Butler object for fetching additional data products before
592  running the Task
593 
594  Returns
595  -------
596  result : `lsst.pipe.base.Struct`
597  Contains whatever additional data the subclass's `run` method needs
598  """
599  pass
600 
601  def getTempExpRefList(self, patchRef, calExpRefList):
602  """Generate list data references corresponding to warped exposures
603  that lie within the patch to be coadded.
604 
605  Parameters
606  ----------
607  patchRef : `dataRef`
608  Data reference for patch.
609  calExpRefList : `list`
610  List of data references for input calexps.
611 
612  Returns
613  -------
614  tempExpRefList : `list`
615  List of Warp/CoaddTempExp data references.
616  """
617  butler = patchRef.getButler()
618  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
619  self.getTempExpDatasetName(self.warpType))
620  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
621  g, groupData.keys) for
622  g in groupData.groups.keys()]
623  return tempExpRefList
624 
625  def prepareInputs(self, refList):
626  """Prepare the input warps for coaddition by measuring the weight for
627  each warp and the scaling for the photometric zero point.
628 
629  Each Warp has its own photometric zeropoint and background variance.
630  Before coadding these Warps together, compute a scale factor to
631  normalize the photometric zeropoint and compute the weight for each Warp.
632 
633  Parameters
634  ----------
635  refList : `list`
636  List of data references to tempExp
637 
638  Returns
639  -------
640  result : `lsst.pipe.base.Struct`
641  Result struct with components:
642 
643  - ``tempExprefList``: `list` of data references to tempExp.
644  - ``weightList``: `list` of weightings.
645  - ``imageScalerList``: `list` of image scalers.
646  """
647  statsCtrl = afwMath.StatisticsControl()
648  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
649  statsCtrl.setNumIter(self.config.clipIter)
650  statsCtrl.setAndMask(self.getBadPixelMask())
651  statsCtrl.setNanSafe(True)
652  # compute tempExpRefList: a list of tempExpRef that actually exist
653  # and weightList: a list of the weight of the associated coadd tempExp
654  # and imageScalerList: a list of scale factors for the associated coadd tempExp
655  tempExpRefList = []
656  weightList = []
657  imageScalerList = []
658  tempExpName = self.getTempExpDatasetName(self.warpType)
659  for tempExpRef in refList:
660  if not tempExpRef.datasetExists(tempExpName):
661  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
662  continue
663 
664  tempExp = tempExpRef.get(tempExpName, immediate=True)
665  # Ignore any input warp that is empty of data
666  if numpy.isnan(tempExp.image.array).all():
667  continue
668  maskedImage = tempExp.getMaskedImage()
669  imageScaler = self.scaleZeroPoint.computeImageScaler(
670  exposure=tempExp,
671  dataRef=tempExpRef,
672  )
673  try:
674  imageScaler.scaleMaskedImage(maskedImage)
675  except Exception as e:
676  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
677  continue
678  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
679  afwMath.MEANCLIP, statsCtrl)
680  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
681  weight = 1.0 / float(meanVar)
682  if not numpy.isfinite(weight):
683  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
684  continue
685  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
686 
687  del maskedImage
688  del tempExp
689 
690  tempExpRefList.append(tempExpRef)
691  weightList.append(weight)
692  imageScalerList.append(imageScaler)
693 
694  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
695  imageScalerList=imageScalerList)
696 
697  def prepareStats(self, mask=None):
698  """Prepare the statistics for coadding images.
699 
700  Parameters
701  ----------
702  mask : `int`, optional
703  Bit mask value to exclude from coaddition.
704 
705  Returns
706  -------
707  stats : `lsst.pipe.base.Struct`
708  Statistics structure with the following fields:
709 
710  - ``statsCtrl``: Statistics control object for coadd
711  (`lsst.afw.math.StatisticsControl`)
712  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
713  """
714  if mask is None:
715  mask = self.getBadPixelMask()
716  statsCtrl = afwMath.StatisticsControl()
717  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
718  statsCtrl.setNumIter(self.config.clipIter)
719  statsCtrl.setAndMask(mask)
720  statsCtrl.setNanSafe(True)
721  statsCtrl.setWeighted(True)
722  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
723  for plane, threshold in self.config.maskPropagationThresholds.items():
724  bit = afwImage.Mask.getMaskPlane(plane)
725  statsCtrl.setMaskPropagationThreshold(bit, threshold)
726  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
727  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
728 
729  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
730  altMaskList=None, mask=None, supplementaryData=None):
731  """Assemble a coadd from input warps
732 
733  Assemble the coadd using the provided list of coaddTempExps. Since
734  the full coadd covers a patch (a large area), the assembly is
735  performed over small areas on the image at a time in order to
736  conserve memory usage. Iterate over subregions within the outer
737  bbox of the patch using `assembleSubregion` to stack the corresponding
738  subregions from the coaddTempExps with the statistic specified.
739  Set the edge bits the coadd mask based on the weight map.
740 
741  Parameters
742  ----------
743  skyInfo : `lsst.pipe.base.Struct`
744  Struct with geometric information about the patch.
745  tempExpRefList : `list`
746  List of data references to Warps (previously called CoaddTempExps).
747  imageScalerList : `list`
748  List of image scalers.
749  weightList : `list`
750  List of weights
751  altMaskList : `list`, optional
752  List of alternate masks to use rather than those stored with
753  tempExp.
754  mask : `int`, optional
755  Bit mask value to exclude from coaddition.
756  supplementaryData : lsst.pipe.base.Struct, optional
757  Struct with additional data products needed to assemble coadd.
758  Only used by subclasses that implement `makeSupplementaryData`
759  and override `run`.
760 
761  Returns
762  -------
763  result : `lsst.pipe.base.Struct`
764  Result struct with components:
765 
766  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
767  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
768  """
769  tempExpName = self.getTempExpDatasetName(self.warpType)
770  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
771  stats = self.prepareStats(mask=mask)
772 
773  if altMaskList is None:
774  altMaskList = [None]*len(tempExpRefList)
775 
776  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
777  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
778  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
779  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
780  coaddMaskedImage = coaddExposure.getMaskedImage()
781  subregionSizeArr = self.config.subregionSize
782  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
783  # if nImage is requested, create a zero one which can be passed to assembleSubregion
784  if self.config.doNImage:
785  nImage = afwImage.ImageU(skyInfo.bbox)
786  else:
787  nImage = None
788  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
789  try:
790  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
791  weightList, altMaskList, stats.flags, stats.ctrl,
792  nImage=nImage)
793  except Exception as e:
794  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
795 
796  self.setInexactPsf(coaddMaskedImage.getMask())
797  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
798  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
799  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
800  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
801 
802  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
803  """Set the metadata for the coadd.
804 
805  This basic implementation sets the filter from the first input.
806 
807  Parameters
808  ----------
809  coaddExposure : `lsst.afw.image.Exposure`
810  The target exposure for the coadd.
811  tempExpRefList : `list`
812  List of data references to tempExp.
813  weightList : `list`
814  List of weights.
815  """
816  assert len(tempExpRefList) == len(weightList), "Length mismatch"
817  tempExpName = self.getTempExpDatasetName(self.warpType)
818  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
819  # (and we need more than just the PropertySet that contains the header), which is not possible
820  # with the current butler (see #2777).
821  tempExpList = [tempExpRef.get(tempExpName + "_sub",
822  bbox=afwGeom.Box2I(coaddExposure.getBBox().getMin(),
823  afwGeom.Extent2I(1, 1)), immediate=True)
824  for tempExpRef in tempExpRefList]
825  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
826 
827  coaddExposure.setFilter(tempExpList[0].getFilter())
828  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
829  coaddInputs.ccds.reserve(numCcds)
830  coaddInputs.visits.reserve(len(tempExpList))
831 
832  for tempExp, weight in zip(tempExpList, weightList):
833  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
834 
835  if self.config.doUsePsfMatchedPolygons:
836  self.shrinkValidPolygons(coaddInputs)
837 
838  coaddInputs.visits.sort()
839  if self.warpType == "psfMatched":
840  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
841  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
842  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
843  # having the maximum width (sufficient because square)
844  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
845  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
846  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
847  else:
848  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
849  self.config.coaddPsf.makeControl())
850  coaddExposure.setPsf(psf)
851  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
852  coaddExposure.getWcs())
853  coaddExposure.getInfo().setApCorrMap(apCorrMap)
854  if self.config.doAttachTransmissionCurve:
855  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
856  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
857 
858  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
859  altMaskList, statsFlags, statsCtrl, nImage=None):
860  """Assemble the coadd for a sub-region.
861 
862  For each coaddTempExp, check for (and swap in) an alternative mask
863  if one is passed. Remove mask planes listed in
864  `config.removeMaskPlanes`. Finally, stack the actual exposures using
865  `lsst.afw.math.statisticsStack` with the statistic specified by
866  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
867  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
868  an N-sigma clipped mean where N and iterations are specified by
869  statsCtrl. Assign the stacked subregion back to the coadd.
870 
871  Parameters
872  ----------
873  coaddExposure : `lsst.afw.image.Exposure`
874  The target exposure for the coadd.
875  bbox : `lsst.afw.geom.Box`
876  Sub-region to coadd.
877  tempExpRefList : `list`
878  List of data reference to tempExp.
879  imageScalerList : `list`
880  List of image scalers.
881  weightList : `list`
882  List of weights.
883  altMaskList : `list`
884  List of alternate masks to use rather than those stored with
885  tempExp, or None. Each element is dict with keys = mask plane
886  name to which to add the spans.
887  statsFlags : `lsst.afw.math.Property`
888  Property object for statistic for coadd.
889  statsCtrl : `lsst.afw.math.StatisticsControl`
890  Statistics control object for coadd.
891  nImage : `lsst.afw.image.ImageU`, optional
892  Keeps track of exposure count for each pixel.
893  """
894  self.log.debug("Computing coadd over %s", bbox)
895  tempExpName = self.getTempExpDatasetName(self.warpType)
896  coaddExposure.mask.addMaskPlane("REJECTED")
897  coaddExposure.mask.addMaskPlane("CLIPPED")
898  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
899  maskMap = self.setRejectedMaskMapping(statsCtrl)
900  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
901  maskedImageList = []
902  if nImage is not None:
903  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
904  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
905  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
906  maskedImage = exposure.getMaskedImage()
907  mask = maskedImage.getMask()
908  if altMask is not None:
909  self.applyAltMaskPlanes(mask, altMask)
910  imageScaler.scaleMaskedImage(maskedImage)
911 
912  # Add 1 for each pixel which is not excluded by the exclude mask.
913  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
914  if nImage is not None:
915  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
916  if self.config.removeMaskPlanes:
917  self.removeMaskPlanes(maskedImage)
918  maskedImageList.append(maskedImage)
919 
920  with self.timer("stack"):
921  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
922  clipped, # also set output to CLIPPED if sigma-clipped
923  maskMap)
924  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
925  if nImage is not None:
926  nImage.assign(subNImage, bbox)
927 
928  def removeMaskPlanes(self, maskedImage):
929  """Unset the mask of an image for mask planes specified in the config.
930 
931  Parameters
932  ----------
933  maskedImage : `lsst.afw.image.MaskedImage`
934  The masked image to be modified.
935  """
936  mask = maskedImage.getMask()
937  for maskPlane in self.config.removeMaskPlanes:
938  try:
939  mask &= ~mask.getPlaneBitMask(maskPlane)
940  except Exception as e:
941  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
942 
943  @staticmethod
944  def setRejectedMaskMapping(statsCtrl):
945  """Map certain mask planes of the warps to new planes for the coadd.
946 
947  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
948  or CLIPPED, set it to REJECTED on the coadd.
949  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
950  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
951 
952  Parameters
953  ----------
954  statsCtrl : `lsst.afw.math.StatisticsControl`
955  Statistics control object for coadd
956 
957  Returns
958  -------
959  maskMap : `list` of `tuple` of `int`
960  A list of mappings of mask planes of the warped exposures to
961  mask planes of the coadd.
962  """
963  edge = afwImage.Mask.getPlaneBitMask("EDGE")
964  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
965  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
966  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
967  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
968  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
969  (clipped, clipped)]
970  return maskMap
971 
972  def applyAltMaskPlanes(self, mask, altMaskSpans):
973  """Apply in place alt mask formatted as SpanSets to a mask.
974 
975  Parameters
976  ----------
977  mask : `lsst.afw.image.Mask`
978  Original mask.
979  altMaskSpans : `dict`
980  SpanSet lists to apply. Each element contains the new mask
981  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
982  and list of SpanSets to apply to the mask.
983 
984  Returns
985  -------
986  mask : `lsst.afw.image.Mask`
987  Updated mask.
988  """
989  if self.config.doUsePsfMatchedPolygons:
990  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
991  # Clear away any other masks outside the validPolygons. These pixels are no longer
992  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
993  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
994  # This mask-clearing step must only occur *before* applying the new masks below
995  for spanSet in altMaskSpans['NO_DATA']:
996  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
997 
998  for plane, spanSetList in altMaskSpans.items():
999  maskClipValue = mask.addMaskPlane(plane)
1000  for spanSet in spanSetList:
1001  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1002  return mask
1003 
1004  def shrinkValidPolygons(self, coaddInputs):
1005  """Shrink coaddInputs' ccds' ValidPolygons in place.
1006 
1007  Either modify each ccd's validPolygon in place, or if CoaddInputs
1008  does not have a validPolygon, create one from its bbox.
1009 
1010  Parameters
1011  ----------
1012  coaddInputs : `lsst.afw.image.coaddInputs`
1013  Original mask.
1014 
1015  """
1016  for ccd in coaddInputs.ccds:
1017  polyOrig = ccd.getValidPolygon()
1018  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1019  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1020  if polyOrig:
1021  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1022  else:
1023  validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
1024  ccd.setValidPolygon(validPolygon)
1025 
1026  def readBrightObjectMasks(self, dataRef):
1027  """Retrieve the bright object masks.
1028 
1029  Returns None on failure.
1030 
1031  Parameters
1032  ----------
1033  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1034  A Butler dataRef.
1035 
1036  Returns
1037  -------
1038  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1039  Bright object mask from the Butler object, or None if it cannot
1040  be retrieved.
1041  """
1042  try:
1043  return dataRef.get("brightObjectMask", immediate=True)
1044  except Exception as e:
1045  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1046  return None
1047 
1048  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
1049  """Set the bright object masks.
1050 
1051  Parameters
1052  ----------
1053  exposure : `lsst.afw.image.Exposure`
1054  Exposure under consideration.
1055  dataId : `lsst.daf.persistence.dataId`
1056  Data identifier dict for patch.
1057  brightObjectMasks : `lsst.afw.table`
1058  Table of bright objects to mask.
1059  """
1060 
1061  if brightObjectMasks is None:
1062  self.log.warn("Unable to apply bright object mask: none supplied")
1063  return
1064  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1065  mask = exposure.getMaskedImage().getMask()
1066  wcs = exposure.getWcs()
1067  plateScale = wcs.getPixelScale().asArcseconds()
1068 
1069  for rec in brightObjectMasks:
1070  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
1071  if rec["type"] == "box":
1072  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1073  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1074  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1075 
1076  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
1077  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
1078 
1079  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1080  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1081  spans = afwGeom.SpanSet(bbox)
1082  elif rec["type"] == "circle":
1083  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1084  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1085  else:
1086  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1087  continue
1088  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1089 
1090  def setInexactPsf(self, mask):
1091  """Set INEXACT_PSF mask plane.
1092 
1093  If any of the input images isn't represented in the coadd (due to
1094  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1095  these pixels.
1096 
1097  Parameters
1098  ----------
1099  mask : `lsst.afw.image.Mask`
1100  Coadded exposure's mask, modified in-place.
1101  """
1102  mask.addMaskPlane("INEXACT_PSF")
1103  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1104  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1105  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1106  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1107  array = mask.getArray()
1108  selected = array & (sensorEdge | clipped | rejected) > 0
1109  array[selected] |= inexactPsf
1110 
1111  @classmethod
1112  def _makeArgumentParser(cls):
1113  """Create an argument parser.
1114  """
1115  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1116  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
1117  cls.ConfigClass().warpType + "Warp",
1118  help="data ID, e.g. --id tract=12345 patch=1,2",
1119  ContainerClass=AssembleCoaddDataIdContainer)
1120  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1121  ContainerClass=SelectDataIdContainer)
1122  return parser
1123 
1124  @staticmethod
1125  def _subBBoxIter(bbox, subregionSize):
1126  """Iterate over subregions of a bbox.
1127 
1128  Parameters
1129  ----------
1130  bbox : `lsst.afw.geom.Box2I`
1131  Bounding box over which to iterate.
1132  subregionSize: `lsst.afw.geom.Extent2I`
1133  Size of sub-bboxes.
1134 
1135  Yields
1136  ------
1137  subBBox : `lsst.afw.geom.Box2I`
1138  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1139  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1140  the edges of ``bbox``, but it will never be empty.
1141  """
1142  if bbox.isEmpty():
1143  raise RuntimeError("bbox %s is empty" % (bbox,))
1144  if subregionSize[0] < 1 or subregionSize[1] < 1:
1145  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1146 
1147  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1148  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1149  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
1150  subBBox.clip(bbox)
1151  if subBBox.isEmpty():
1152  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1153  "colShift=%s, rowShift=%s" %
1154  (bbox, subregionSize, colShift, rowShift))
1155  yield subBBox
1156 
1157 
1158 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1159  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1160  """
1161 
1162  def makeDataRefList(self, namespace):
1163  """Make self.refList from self.idList.
1164 
1165  Parameters
1166  ----------
1167  namespace
1168  Results of parsing command-line (with ``butler`` and ``log`` elements).
1169  """
1170  datasetType = namespace.config.coaddName + "Coadd"
1171  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1172 
1173  for dataId in self.idList:
1174  # tract and patch are required
1175  for key in keysCoadd:
1176  if key not in dataId:
1177  raise RuntimeError("--id must include " + key)
1178 
1179  dataRef = namespace.butler.dataRef(
1180  datasetType=datasetType,
1181  dataId=dataId,
1182  )
1183  self.refList.append(dataRef)
1184 
1185 
1186 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1187  """Function to count the number of pixels with a specific mask in a
1188  footprint.
1189 
1190  Find the intersection of mask & footprint. Count all pixels in the mask
1191  that are in the intersection that have bitmask set but do not have
1192  ignoreMask set. Return the count.
1193 
1194  Parameters
1195  ----------
1196  mask : `lsst.afw.image.Mask`
1197  Mask to define intersection region by.
1198  footprint : `lsst.afw.detection.Footprint`
1199  Footprint to define the intersection region by.
1200  bitmask
1201  Specific mask that we wish to count the number of occurances of.
1202  ignoreMask
1203  Pixels to not consider.
1204 
1205  Returns
1206  -------
1207  result : `int`
1208  Count of number of pixels in footprint with specified mask.
1209  """
1210  bbox = footprint.getBBox()
1211  bbox.clip(mask.getBBox(afwImage.PARENT))
1212  fp = afwImage.Mask(bbox)
1213  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1214  footprint.spans.setMask(fp, bitmask)
1215  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1216  (subMask.getArray() & ignoreMask) == 0).sum()
1217 
1218 
1220  """Configuration parameters for the SafeClipAssembleCoaddTask.
1221  """
1222  clipDetection = pexConfig.ConfigurableField(
1223  target=SourceDetectionTask,
1224  doc="Detect sources on difference between unclipped and clipped coadd")
1225  minClipFootOverlap = pexConfig.Field(
1226  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1227  dtype=float,
1228  default=0.6
1229  )
1230  minClipFootOverlapSingle = pexConfig.Field(
1231  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1232  "clipped when only one visit overlaps",
1233  dtype=float,
1234  default=0.5
1235  )
1236  minClipFootOverlapDouble = pexConfig.Field(
1237  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1238  "clipped when two visits overlap",
1239  dtype=float,
1240  default=0.45
1241  )
1242  maxClipFootOverlapDouble = pexConfig.Field(
1243  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1244  "considering two visits",
1245  dtype=float,
1246  default=0.15
1247  )
1248  minBigOverlap = pexConfig.Field(
1249  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1250  "when labeling clipped footprints",
1251  dtype=int,
1252  default=100
1253  )
1254 
1255  def setDefaults(self):
1256  """Set default values for clipDetection.
1257 
1258  Notes
1259  -----
1260  The numeric values for these configuration parameters were
1261  empirically determined, future work may further refine them.
1262  """
1263  AssembleCoaddConfig.setDefaults(self)
1264  self.clipDetection.doTempLocalBackground = False
1265  self.clipDetection.reEstimateBackground = False
1266  self.clipDetection.returnOriginalFootprints = False
1267  self.clipDetection.thresholdPolarity = "both"
1268  self.clipDetection.thresholdValue = 2
1269  self.clipDetection.nSigmaToGrow = 2
1270  self.clipDetection.minPixels = 4
1271  self.clipDetection.isotropicGrow = True
1272  self.clipDetection.thresholdType = "pixel_stdev"
1273  self.sigmaClip = 1.5
1274  self.clipIter = 3
1275  self.statistic = "MEAN"
1276 
1277  def validate(self):
1278  if self.doSigmaClip:
1279  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1280  "Ignoring doSigmaClip.")
1281  self.doSigmaClip = False
1282  if self.statistic != "MEAN":
1283  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1284  "(%s chosen). Please set statistic to MEAN."
1285  % (self.statistic))
1286  AssembleCoaddTask.ConfigClass.validate(self)
1287 
1288 
1290  """Assemble a coadded image from a set of coadded temporary exposures,
1291  being careful to clip & flag areas with potential artifacts.
1292 
1293  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1294  we clip outliers). The problem with doing this is that when computing the
1295  coadd PSF at a given location, individual visit PSFs from visits with
1296  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1297  In this task, we correct for this behavior by creating a new
1298  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1299  coaddTempExps and the final coadd where
1300 
1301  i. difference imaging suggests that there is an outlier and
1302  ii. this outlier appears on only one or two images.
1303 
1304  Such regions will not contribute to the final coadd. Furthermore, any
1305  routine to determine the coadd PSF can now be cognizant of clipped regions.
1306  Note that the algorithm implemented by this task is preliminary and works
1307  correctly for HSC data. Parameter modifications and or considerable
1308  redesigning of the algorithm is likley required for other surveys.
1309 
1310  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1311  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1312  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1313  if you wish.
1314 
1315  Notes
1316  -----
1317  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1318  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1319  see `baseDebug` for more about ``debug.py`` files.
1320  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1321  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1322  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1323  for further information.
1324 
1325  Examples
1326  --------
1327  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1328  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1329  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1330 
1331  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1332  and filter to be coadded (specified using
1333  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1334  along with a list of coaddTempExps to attempt to coadd (specified using
1335  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1336  Only the coaddTempExps that cover the specified tract and patch will be
1337  coadded. A list of the available optional arguments can be obtained by
1338  calling assembleCoadd.py with the --help command line argument:
1339 
1340  .. code-block:: none
1341 
1342  assembleCoadd.py --help
1343 
1344  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1345  context of multi-band processing, we will generate the HSC-I & -R band
1346  coadds from HSC engineering test data provided in the ci_hsc package.
1347  To begin, assuming that the lsst stack has been already set up, we must
1348  set up the obs_subaru and ci_hsc packages. This defines the environment
1349  variable $CI_HSC_DIR and points at the location of the package. The raw
1350  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1351  the coadds, we must first
1352 
1353  - ``processCcd``
1354  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1355  - ``makeSkyMap``
1356  create a skymap that covers the area of the sky present in the raw exposures
1357  - ``makeCoaddTempExp``
1358  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1359 
1360  We can perform all of these steps by running
1361 
1362  .. code-block:: none
1363 
1364  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1365 
1366  This will produce warped coaddTempExps for each visit. To coadd the
1367  warped data, we call ``assembleCoadd.py`` as follows:
1368 
1369  .. code-block:: none
1370 
1371  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1372  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1373  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1374  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1375  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1376  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1377  --selectId visit=903988 ccd=24
1378 
1379  This will process the HSC-I band data. The results are written in
1380  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1381 
1382  You may also choose to run:
1383 
1384  .. code-block:: none
1385 
1386  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1387  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1388  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1389  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1390  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1391  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1392  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1393  --selectId visit=903346 ccd=12
1394 
1395  to generate the coadd for the HSC-R band if you are interested in following
1396  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1397  """
1398  ConfigClass = SafeClipAssembleCoaddConfig
1399  _DefaultName = "safeClipAssembleCoadd"
1400 
1401  def __init__(self, *args, **kwargs):
1402  AssembleCoaddTask.__init__(self, *args, **kwargs)
1403  schema = afwTable.SourceTable.makeMinimalSchema()
1404  self.makeSubtask("clipDetection", schema=schema)
1405 
1406  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1407  """Assemble the coadd for a region.
1408 
1409  Compute the difference of coadds created with and without outlier
1410  rejection to identify coadd pixels that have outlier values in some
1411  individual visits.
1412  Detect clipped regions on the difference image and mark these regions
1413  on the one or two individual coaddTempExps where they occur if there
1414  is significant overlap between the clipped region and a source. This
1415  leaves us with a set of footprints from the difference image that have
1416  been identified as having occured on just one or two individual visits.
1417  However, these footprints were generated from a difference image. It
1418  is conceivable for a large diffuse source to have become broken up
1419  into multiple footprints acrosss the coadd difference in this process.
1420  Determine the clipped region from all overlapping footprints from the
1421  detected sources in each visit - these are big footprints.
1422  Combine the small and big clipped footprints and mark them on a new
1423  bad mask plane.
1424  Generate the coadd using `AssembleCoaddTask.run` without outlier
1425  removal. Clipped footprints will no longer make it into the coadd
1426  because they are marked in the new bad mask plane.
1427 
1428  Parameters
1429  ----------
1430  skyInfo : `lsst.pipe.base.Struct`
1431  Patch geometry information, from getSkyInfo
1432  tempExpRefList : `list`
1433  List of data reference to tempExp
1434  imageScalerList : `list`
1435  List of image scalers
1436  weightList : `list`
1437  List of weights
1438 
1439  Returns
1440  -------
1441  result : `lsst.pipe.base.Struct`
1442  Result struct with components:
1443 
1444  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1445  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
1446 
1447  Notes
1448  -----
1449  args and kwargs are passed but ignored in order to match the call
1450  signature expected by the parent task.
1451  """
1452  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1453  mask = exp.getMaskedImage().getMask()
1454  mask.addMaskPlane("CLIPPED")
1455 
1456  result = self.detectClip(exp, tempExpRefList)
1457 
1458  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1459 
1460  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1461  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1462  # Append big footprints from individual Warps to result.clipSpans
1463  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1464  result.detectionFootprints, maskClipValue, maskDetValue,
1465  exp.getBBox())
1466  # Create mask of the current clipped footprints
1467  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1468  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1469 
1470  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1471  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1472  maskClip |= maskClipBig
1473 
1474  # Assemble coadd from base class, but ignoring CLIPPED pixels
1475  badMaskPlanes = self.config.badMaskPlanes[:]
1476  badMaskPlanes.append("CLIPPED")
1477  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1478  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1479  result.clipSpans, mask=badPixelMask)
1480 
1481  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1482  """Return an exposure that contains the difference between unclipped
1483  and clipped coadds.
1484 
1485  Generate a difference image between clipped and unclipped coadds.
1486  Compute the difference image by subtracting an outlier-clipped coadd
1487  from an outlier-unclipped coadd. Return the difference image.
1488 
1489  Parameters
1490  ----------
1491  skyInfo : `lsst.pipe.base.Struct`
1492  Patch geometry information, from getSkyInfo
1493  tempExpRefList : `list`
1494  List of data reference to tempExp
1495  imageScalerList : `list`
1496  List of image scalers
1497  weightList : `list`
1498  List of weights
1499 
1500  Returns
1501  -------
1502  exp : `lsst.afw.image.Exposure`
1503  Difference image of unclipped and clipped coadd wrapped in an Exposure
1504  """
1505  # Clone and upcast self.config because current self.config is frozen
1506  config = AssembleCoaddConfig()
1507  # getattr necessary because subtasks do not survive Config.toDict()
1508  configIntersection = {k: getattr(self.config, k)
1509  for k, v in self.config.toDict().items() if (k in config.keys())}
1510  config.update(**configIntersection)
1511 
1512  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1513  config.statistic = 'MEAN'
1514  task = AssembleCoaddTask(config=config)
1515  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1516 
1517  config.statistic = 'MEANCLIP'
1518  task = AssembleCoaddTask(config=config)
1519  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1520 
1521  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1522  coaddDiff -= coaddClip.getMaskedImage()
1523  exp = afwImage.ExposureF(coaddDiff)
1524  exp.setPsf(coaddMean.getPsf())
1525  return exp
1526 
1527  def detectClip(self, exp, tempExpRefList):
1528  """Detect clipped regions on an exposure and set the mask on the
1529  individual tempExp masks.
1530 
1531  Detect footprints in the difference image after smoothing the
1532  difference image with a Gaussian kernal. Identify footprints that
1533  overlap with one or two input ``coaddTempExps`` by comparing the
1534  computed overlap fraction to thresholds set in the config. A different
1535  threshold is applied depending on the number of overlapping visits
1536  (restricted to one or two). If the overlap exceeds the thresholds,
1537  the footprint is considered "CLIPPED" and is marked as such on the
1538  coaddTempExp. Return a struct with the clipped footprints, the indices
1539  of the ``coaddTempExps`` that end up overlapping with the clipped
1540  footprints, and a list of new masks for the ``coaddTempExps``.
1541 
1542  Parameters
1543  ----------
1544  exp : `lsst.afw.image.Exposure`
1545  Exposure to run detection on.
1546  tempExpRefList : `list`
1547  List of data reference to tempExp.
1548 
1549  Returns
1550  -------
1551  result : `lsst.pipe.base.Struct`
1552  Result struct with components:
1553 
1554  - ``clipFootprints``: list of clipped footprints.
1555  - ``clipIndices``: indices for each ``clippedFootprint`` in
1556  ``tempExpRefList``.
1557  - ``clipSpans``: List of dictionaries containing spanSet lists
1558  to clip. Each element contains the new maskplane name
1559  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1560  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1561  compressed into footprints.
1562  """
1563  mask = exp.getMaskedImage().getMask()
1564  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1565  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1566  # Merge positive and negative together footprints together
1567  fpSet.positive.merge(fpSet.negative)
1568  footprints = fpSet.positive
1569  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1570  ignoreMask = self.getBadPixelMask()
1571 
1572  clipFootprints = []
1573  clipIndices = []
1574  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1575 
1576  # for use by detectClipBig
1577  visitDetectionFootprints = []
1578 
1579  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1580  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1581  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1582 
1583  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1584  for i, warpRef in enumerate(tempExpRefList):
1585  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1586  immediate=True).getMaskedImage().getMask()
1587  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1588  afwImage.PARENT, True)
1589  maskVisitDet &= maskDetValue
1590  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1591  visitDetectionFootprints.append(visitFootprints)
1592 
1593  for j, footprint in enumerate(footprints.getFootprints()):
1594  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1595  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1596 
1597  # build a list of clipped spans for each visit
1598  for j, footprint in enumerate(footprints.getFootprints()):
1599  nPixel = footprint.getArea()
1600  overlap = [] # hold the overlap with each visit
1601  indexList = [] # index of visit in global list
1602  for i in range(len(tempExpRefList)):
1603  ignore = ignoreArr[i, j]
1604  overlapDet = overlapDetArr[i, j]
1605  totPixel = nPixel - ignore
1606 
1607  # If we have more bad pixels than detection skip
1608  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1609  continue
1610  overlap.append(overlapDet/float(totPixel))
1611  indexList.append(i)
1612 
1613  overlap = numpy.array(overlap)
1614  if not len(overlap):
1615  continue
1616 
1617  keep = False # Should this footprint be marked as clipped?
1618  keepIndex = [] # Which tempExps does the clipped footprint belong to
1619 
1620  # If footprint only has one overlap use a lower threshold
1621  if len(overlap) == 1:
1622  if overlap[0] > self.config.minClipFootOverlapSingle:
1623  keep = True
1624  keepIndex = [0]
1625  else:
1626  # This is the general case where only visit should be clipped
1627  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1628  if len(clipIndex) == 1:
1629  keep = True
1630  keepIndex = [clipIndex[0]]
1631 
1632  # Test if there are clipped objects that overlap two different visits
1633  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1634  if len(clipIndex) == 2 and len(overlap) > 3:
1635  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1636  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1637  keep = True
1638  keepIndex = clipIndex
1639 
1640  if not keep:
1641  continue
1642 
1643  for index in keepIndex:
1644  globalIndex = indexList[index]
1645  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1646 
1647  clipIndices.append(numpy.array(indexList)[keepIndex])
1648  clipFootprints.append(footprint)
1649 
1650  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1651  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1652 
1653  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1654  maskClipValue, maskDetValue, coaddBBox):
1655  """Return individual warp footprints for large artifacts and append
1656  them to ``clipList`` in place.
1657 
1658  Identify big footprints composed of many sources in the coadd
1659  difference that may have originated in a large diffuse source in the
1660  coadd. We do this by indentifying all clipped footprints that overlap
1661  significantly with each source in all the coaddTempExps.
1662 
1663  Parameters
1664  ----------
1665  clipList : `list`
1666  List of alt mask SpanSets with clipping information. Modified.
1667  clipFootprints : `list`
1668  List of clipped footprints.
1669  clipIndices : `list`
1670  List of which entries in tempExpClipList each footprint belongs to.
1671  maskClipValue
1672  Mask value of clipped pixels.
1673  maskDetValue
1674  Mask value of detected pixels.
1675  coaddBBox : `lsst.afw.geom.Box`
1676  BBox of the coadd and warps.
1677 
1678  Returns
1679  -------
1680  bigFootprintsCoadd : `list`
1681  List of big footprints
1682  """
1683  bigFootprintsCoadd = []
1684  ignoreMask = self.getBadPixelMask()
1685  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1686  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1687  for footprint in visitFootprints.getFootprints():
1688  footprint.spans.setMask(maskVisitDet, maskDetValue)
1689 
1690  # build a mask of clipped footprints that are in this visit
1691  clippedFootprintsVisit = []
1692  for foot, clipIndex in zip(clipFootprints, clipIndices):
1693  if index not in clipIndex:
1694  continue
1695  clippedFootprintsVisit.append(foot)
1696  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1697  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1698 
1699  bigFootprintsVisit = []
1700  for foot in visitFootprints.getFootprints():
1701  if foot.getArea() < self.config.minBigOverlap:
1702  continue
1703  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1704  if nCount > self.config.minBigOverlap:
1705  bigFootprintsVisit.append(foot)
1706  bigFootprintsCoadd.append(foot)
1707 
1708  for footprint in bigFootprintsVisit:
1709  clippedSpans["CLIPPED"].append(footprint.spans)
1710 
1711  return bigFootprintsCoadd
1712 
1713 
1715  assembleStaticSkyModel = pexConfig.ConfigurableField(
1716  target=AssembleCoaddTask,
1717  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1718  " naive/first-iteration model of the static sky.",
1719  )
1720  detect = pexConfig.ConfigurableField(
1721  target=SourceDetectionTask,
1722  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1723  )
1724  detectTemplate = pexConfig.ConfigurableField(
1725  target=SourceDetectionTask,
1726  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1727  )
1728  maxNumEpochs = pexConfig.Field(
1729  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1730  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1731  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1732  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1733  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1734  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1735  "than transient and not masked.",
1736  dtype=int,
1737  default=2
1738  )
1739  maxFractionEpochsLow = pexConfig.RangeField(
1740  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1741  "Effective maxNumEpochs = "
1742  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1743  dtype=float,
1744  default=0.4,
1745  min=0., max=1.,
1746  )
1747  maxFractionEpochsHigh = pexConfig.RangeField(
1748  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1749  "Effective maxNumEpochs = "
1750  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1751  dtype=float,
1752  default=0.03,
1753  min=0., max=1.,
1754  )
1755  spatialThreshold = pexConfig.RangeField(
1756  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1757  "temporal criteria. If 0, clip all. If 1, clip none.",
1758  dtype=float,
1759  default=0.5,
1760  min=0., max=1.,
1761  inclusiveMin=True, inclusiveMax=True
1762  )
1763  doScaleWarpVariance = pexConfig.Field(
1764  doc="Rescale Warp variance plane using empirical noise?",
1765  dtype=bool,
1766  default=True,
1767  )
1768  scaleWarpVariance = pexConfig.ConfigurableField(
1769  target=ScaleVarianceTask,
1770  doc="Rescale variance on warps",
1771  )
1772  doPreserveContainedBySource = pexConfig.Field(
1773  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1774  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1775  dtype=bool,
1776  default=True,
1777  )
1778  doPrefilterArtifacts = pexConfig.Field(
1779  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1780  "because they will be excluded anyway. This prevents them from contributing "
1781  "to the outlier epoch count image and potentially being labeled as persistant."
1782  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1783  dtype=bool,
1784  default=True
1785  )
1786  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1787  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1788  dtype=str,
1789  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1790  )
1791  prefilterArtifactsRatio = pexConfig.Field(
1792  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1793  dtype=float,
1794  default=0.05
1795  )
1796  psfMatchedWarps = pipeBase.InputDatasetField(
1797  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1798  "Only PSF-Matched Warps make sense for image subtraction. "
1799  "Therefore, they must be in the InputDatasetField and made available to the task."),
1800  nameTemplate="{inputCoaddName}Coadd_psfMatchedWarp",
1801  storageClass="ExposureF",
1802  dimensions=("Tract", "Patch", "SkyMap", "Visit"),
1803  manualLoad=True
1804  )
1805 
1806  def setDefaults(self):
1807  AssembleCoaddConfig.setDefaults(self)
1808  self.statistic = 'MEAN'
1810 
1811  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1812  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1813  if "EDGE" in self.badMaskPlanes:
1814  self.badMaskPlanes.remove('EDGE')
1815  self.removeMaskPlanes.append('EDGE')
1816  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1817  self.assembleStaticSkyModel.warpType = 'psfMatched'
1818  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1819  self.assembleStaticSkyModel.sigmaClip = 2.5
1820  self.assembleStaticSkyModel.clipIter = 3
1821  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1822  self.assembleStaticSkyModel.doWrite = False
1823  self.detect.doTempLocalBackground = False
1824  self.detect.reEstimateBackground = False
1825  self.detect.returnOriginalFootprints = False
1826  self.detect.thresholdPolarity = "both"
1827  self.detect.thresholdValue = 5
1828  self.detect.nSigmaToGrow = 2
1829  self.detect.minPixels = 4
1830  self.detect.isotropicGrow = True
1831  self.detect.thresholdType = "pixel_stdev"
1832  self.detectTemplate.nSigmaToGrow = 2.0
1833  self.detectTemplate.doTempLocalBackground = False
1834  self.detectTemplate.reEstimateBackground = False
1835  self.detectTemplate.returnOriginalFootprints = False
1836 
1837 
1839  """Assemble a compareWarp coadded image from a set of warps
1840  by masking artifacts detected by comparing PSF-matched warps.
1841 
1842  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1843  we clip outliers). The problem with doing this is that when computing the
1844  coadd PSF at a given location, individual visit PSFs from visits with
1845  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1846  In this task, we correct for this behavior by creating a new badMaskPlane
1847  'CLIPPED' which marks pixels in the individual warps suspected to contain
1848  an artifact. We populate this plane on the input warps by comparing
1849  PSF-matched warps with a PSF-matched median coadd which serves as a
1850  model of the static sky. Any group of pixels that deviates from the
1851  PSF-matched template coadd by more than config.detect.threshold sigma,
1852  is an artifact candidate. The candidates are then filtered to remove
1853  variable sources and sources that are difficult to subtract such as
1854  bright stars. This filter is configured using the config parameters
1855  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1856  the maximum fraction of epochs that the deviation can appear in and still
1857  be considered an artifact. The spatialThreshold is the maximum fraction of
1858  pixels in the footprint of the deviation that appear in other epochs
1859  (where other epochs is defined by the temporalThreshold). If the deviant
1860  region meets this criteria of having a significant percentage of pixels
1861  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1862  set in the mask. These regions will not contribute to the final coadd.
1863  Furthermore, any routine to determine the coadd PSF can now be cognizant
1864  of clipped regions. Note that the algorithm implemented by this task is
1865  preliminary and works correctly for HSC data. Parameter modifications and
1866  or considerable redesigning of the algorithm is likley required for other
1867  surveys.
1868 
1869  ``CompareWarpAssembleCoaddTask`` sub-classes
1870  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1871  as a subtask to generate the TemplateCoadd (the model of the static sky).
1872 
1873  Notes
1874  -----
1875  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1876  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1877  ``baseDebug`` for more about ``debug.py`` files.
1878 
1879  This task supports the following debug variables:
1880 
1881  - ``saveCountIm``
1882  If True then save the Epoch Count Image as a fits file in the `figPath`
1883  - ``figPath``
1884  Path to save the debug fits images and figures
1885 
1886  For example, put something like:
1887 
1888  .. code-block:: python
1889 
1890  import lsstDebug
1891  def DebugInfo(name):
1892  di = lsstDebug.getInfo(name)
1893  if name == "lsst.pipe.tasks.assembleCoadd":
1894  di.saveCountIm = True
1895  di.figPath = "/desired/path/to/debugging/output/images"
1896  return di
1897  lsstDebug.Info = DebugInfo
1898 
1899  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1900  ``--debug`` flag. Some subtasks may have their own debug variables;
1901  see individual Task documentation.
1902 
1903  Examples
1904  --------
1905  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1906  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1907  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1908  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1909  and filter to be coadded (specified using
1910  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1911  along with a list of coaddTempExps to attempt to coadd (specified using
1912  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1913  Only the warps that cover the specified tract and patch will be coadded.
1914  A list of the available optional arguments can be obtained by calling
1915  ``assembleCoadd.py`` with the ``--help`` command line argument:
1916 
1917  .. code-block:: none
1918 
1919  assembleCoadd.py --help
1920 
1921  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1922  context of multi-band processing, we will generate the HSC-I & -R band
1923  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1924  To begin, assuming that the lsst stack has been already set up, we must
1925  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1926  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1927  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1928  directory. To begin assembling the coadds, we must first
1929 
1930  - processCcd
1931  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1932  - makeSkyMap
1933  create a skymap that covers the area of the sky present in the raw exposures
1934  - makeCoaddTempExp
1935  warp the individual calibrated exposures to the tangent plane of the coadd
1936 
1937  We can perform all of these steps by running
1938 
1939  .. code-block:: none
1940 
1941  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1942 
1943  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1944  warped data, we call ``assembleCoadd.py`` as follows:
1945 
1946  .. code-block:: none
1947 
1948  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1949  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1950  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1951  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1952  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1953  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1954  --selectId visit=903988 ccd=24
1955 
1956  This will process the HSC-I band data. The results are written in
1957  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1958  """
1959  ConfigClass = CompareWarpAssembleCoaddConfig
1960  _DefaultName = "compareWarpAssembleCoadd"
1961 
1962  def __init__(self, *args, **kwargs):
1963  AssembleCoaddTask.__init__(self, *args, **kwargs)
1964  self.makeSubtask("assembleStaticSkyModel")
1965  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1966  self.makeSubtask("detect", schema=detectionSchema)
1967  if self.config.doPreserveContainedBySource:
1968  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1969  if self.config.doScaleWarpVariance:
1970  self.makeSubtask("scaleWarpVariance")
1971 
1972  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
1973  """Make inputs specific to Subclass with Gen 3 API
1974 
1975  Calls Gen3 `adaptArgsAndRun` instead of the Gen2 specific `runDataRef`
1976 
1977  Duplicates interface of`adaptArgsAndRun` method.
1978  Available to be implemented by subclasses only if they need the
1979  coadd dataRef for performing preliminary processing before
1980  assembling the coadd.
1981 
1982  Parameters
1983  ----------
1984  inputData : `dict`
1985  Keys are the names of the configs describing input dataset types.
1986  Values are input Python-domain data objects (or lists of objects)
1987  retrieved from data butler.
1988  inputDataIds : `dict`
1989  Keys are the names of the configs describing input dataset types.
1990  Values are DataIds (or lists of DataIds) that task consumes for
1991  corresponding dataset type.
1992  DataIds are guaranteed to match data objects in ``inputData``.
1993  outputDataIds : `dict`
1994  Keys are the names of the configs describing input dataset types.
1995  Values are DataIds (or lists of DataIds) that task is to produce
1996  for corresponding dataset type.
1997  butler : `lsst.daf.butler.Butler`
1998  Gen3 Butler object for fetching additional data products before
1999  running the Task
2000 
2001  Returns
2002  -------
2003  result : `lsst.pipe.base.Struct`
2004  Result struct with components:
2005 
2006  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2007  - ``nImage``: N Image (``lsst.afw.image.Image``)
2008 
2009 
2010  """
2011  templateCoadd = self.assembleStaticSkyModel.adaptArgsAndRun(inputData, inputDataIds,
2012  outputDataIds, butler)
2013  if templateCoadd is None:
2014  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2015 
2016  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2017  nImage=templateCoadd.nImage)
2018 
2019  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2020  """Make inputs specific to Subclass.
2021 
2022  Generate a templateCoadd to use as a native model of static sky to
2023  subtract from warps.
2024 
2025  Parameters
2026  ----------
2027  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2028  Butler dataRef for supplementary data.
2029  selectDataList : `list` (optional)
2030  Optional List of data references to Calexps.
2031  warpRefList : `list` (optional)
2032  Optional List of data references to Warps.
2033 
2034  Returns
2035  -------
2036  result : `lsst.pipe.base.Struct`
2037  Result struct with components:
2038 
2039  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2040  - ``nImage``: N Image (``lsst.afw.image.Image``)
2041  """
2042  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2043  if templateCoadd is None:
2044  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2045 
2046  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2047  nImage=templateCoadd.nImage)
2048 
2049  def _noTemplateMessage(self, warpType):
2050  warpName = (warpType[0].upper() + warpType[1:])
2051  message = """No %(warpName)s warps were found to build the template coadd which is
2052  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2053  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2054  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2055 
2056  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2057  another algorithm like:
2058 
2059  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2060  config.assemble.retarget(SafeClipAssembleCoaddTask)
2061  """ % {"warpName": warpName}
2062  return message
2063 
2064  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2065  supplementaryData, *args, **kwargs):
2066  """Assemble the coadd.
2067 
2068  Find artifacts and apply them to the warps' masks creating a list of
2069  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2070  plane. Then pass these alternative masks to the base class's `run`
2071  method.
2072 
2073  Parameters
2074  ----------
2075  skyInfo : `lsst.pipe.base.Struct`
2076  Patch geometry information.
2077  tempExpRefList : `list`
2078  List of data references to warps.
2079  imageScalerList : `list`
2080  List of image scalers.
2081  weightList : `list`
2082  List of weights.
2083  supplementaryData : `lsst.pipe.base.Struct`
2084  This Struct must contain a ``templateCoadd`` that serves as the
2085  model of the static sky.
2086 
2087  Returns
2088  -------
2089  result : `lsst.pipe.base.Struct`
2090  Result struct with components:
2091 
2092  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
2093  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
2094  """
2095  templateCoadd = supplementaryData.templateCoadd
2096  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
2097  badMaskPlanes = self.config.badMaskPlanes[:]
2098  badMaskPlanes.append("CLIPPED")
2099  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2100 
2101  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2102  spanSetMaskList, mask=badPixelMask)
2103 
2104  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2105  # Psf-Matching moves the real edge inwards
2106  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2107  return result
2108 
2109  def applyAltEdgeMask(self, mask, altMaskList):
2110  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2111 
2112  Parameters
2113  ----------
2114  mask : `lsst.afw.image.Mask`
2115  Original mask.
2116  altMaskList : `list`
2117  List of Dicts containing ``spanSet`` lists.
2118  Each element contains the new mask plane name (e.g. "CLIPPED
2119  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2120  the mask.
2121  """
2122  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2123  for visitMask in altMaskList:
2124  if "EDGE" in visitMask:
2125  for spanSet in visitMask['EDGE']:
2126  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2127 
2128  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2129  """Find artifacts.
2130 
2131  Loop through warps twice. The first loop builds a map with the count
2132  of how many epochs each pixel deviates from the templateCoadd by more
2133  than ``config.chiThreshold`` sigma. The second loop takes each
2134  difference image and filters the artifacts detected in each using
2135  count map to filter out variable sources and sources that are
2136  difficult to subtract cleanly.
2137 
2138  Parameters
2139  ----------
2140  templateCoadd : `lsst.afw.image.Exposure`
2141  Exposure to serve as model of static sky.
2142  tempExpRefList : `list`
2143  List of data references to warps.
2144  imageScalerList : `list`
2145  List of image scalers.
2146 
2147  Returns
2148  -------
2149  altMasks : `list`
2150  List of dicts containing information about CLIPPED
2151  (i.e., artifacts), NO_DATA, and EDGE pixels.
2152  """
2153 
2154  self.log.debug("Generating Count Image, and mask lists.")
2155  coaddBBox = templateCoadd.getBBox()
2156  slateIm = afwImage.ImageU(coaddBBox)
2157  epochCountImage = afwImage.ImageU(coaddBBox)
2158  nImage = afwImage.ImageU(coaddBBox)
2159  spanSetArtifactList = []
2160  spanSetNoDataMaskList = []
2161  spanSetEdgeList = []
2162  badPixelMask = self.getBadPixelMask()
2163 
2164  # mask of the warp diffs should = that of only the warp
2165  templateCoadd.mask.clearAllMaskPlanes()
2166 
2167  if self.config.doPreserveContainedBySource:
2168  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2169  else:
2170  templateFootprints = None
2171 
2172  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2173  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2174  if warpDiffExp is not None:
2175  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2176  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2177  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2178  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2179  fpSet.positive.merge(fpSet.negative)
2180  footprints = fpSet.positive
2181  slateIm.set(0)
2182  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2183 
2184  # Remove artifacts due to defects before they contribute to the epochCountImage
2185  if self.config.doPrefilterArtifacts:
2186  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2187  for spans in spanSetList:
2188  spans.setImage(slateIm, 1, doClip=True)
2189  epochCountImage += slateIm
2190 
2191  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2192  # undergo a second convolution. Pixels with data in the direct warp
2193  # but not in the PSF-matched warp will not have their artifacts detected.
2194  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2195  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2196  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2197  nansMask.setXY0(warpDiffExp.getXY0())
2198  edgeMask = warpDiffExp.mask
2199  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2200  edgeMask.getPlaneBitMask("EDGE")).split()
2201  else:
2202  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2203  # In this case, mask the whole epoch
2204  nansMask = afwImage.MaskX(coaddBBox, 1)
2205  spanSetList = []
2206  spanSetEdgeMask = []
2207 
2208  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2209 
2210  spanSetNoDataMaskList.append(spanSetNoDataMask)
2211  spanSetArtifactList.append(spanSetList)
2212  spanSetEdgeList.append(spanSetEdgeMask)
2213 
2214  if lsstDebug.Info(__name__).saveCountIm:
2215  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2216  epochCountImage.writeFits(path)
2217 
2218  for i, spanSetList in enumerate(spanSetArtifactList):
2219  if spanSetList:
2220  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2221  templateFootprints)
2222  spanSetArtifactList[i] = filteredSpanSetList
2223 
2224  altMasks = []
2225  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2226  altMasks.append({'CLIPPED': artifacts,
2227  'NO_DATA': noData,
2228  'EDGE': edge})
2229  return altMasks
2230 
2231  def prefilterArtifacts(self, spanSetList, exp):
2232  """Remove artifact candidates covered by bad mask plane.
2233 
2234  Any future editing of the candidate list that does not depend on
2235  temporal information should go in this method.
2236 
2237  Parameters
2238  ----------
2239  spanSetList : `list`
2240  List of SpanSets representing artifact candidates.
2241  exp : `lsst.afw.image.Exposure`
2242  Exposure containing mask planes used to prefilter.
2243 
2244  Returns
2245  -------
2246  returnSpanSetList : `list`
2247  List of SpanSets with artifacts.
2248  """
2249  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2250  goodArr = (exp.mask.array & badPixelMask) == 0
2251  returnSpanSetList = []
2252  bbox = exp.getBBox()
2253  x0, y0 = exp.getXY0()
2254  for i, span in enumerate(spanSetList):
2255  y, x = span.clippedTo(bbox).indices()
2256  yIndexLocal = numpy.array(y) - y0
2257  xIndexLocal = numpy.array(x) - x0
2258  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2259  if goodRatio > self.config.prefilterArtifactsRatio:
2260  returnSpanSetList.append(span)
2261  return returnSpanSetList
2262 
2263  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2264  """Filter artifact candidates.
2265 
2266  Parameters
2267  ----------
2268  spanSetList : `list`
2269  List of SpanSets representing artifact candidates.
2270  epochCountImage : `lsst.afw.image.Image`
2271  Image of accumulated number of warpDiff detections.
2272  nImage : `lsst.afw.image.Image`
2273  Image of the accumulated number of total epochs contributing.
2274 
2275  Returns
2276  -------
2277  maskSpanSetList : `list`
2278  List of SpanSets with artifacts.
2279  """
2280 
2281  maskSpanSetList = []
2282  x0, y0 = epochCountImage.getXY0()
2283  for i, span in enumerate(spanSetList):
2284  y, x = span.indices()
2285  yIdxLocal = [y1 - y0 for y1 in y]
2286  xIdxLocal = [x1 - x0 for x1 in x]
2287  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2288  totalN = nImage.array[yIdxLocal, xIdxLocal]
2289 
2290  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2291  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2292  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2293  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2294  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2295  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2296  (outlierN <= effectiveMaxNumEpochs))
2297  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2298  if percentBelowThreshold > self.config.spatialThreshold:
2299  maskSpanSetList.append(span)
2300 
2301  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2302  # If a candidate is contained by a footprint on the template coadd, do not clip
2303  filteredMaskSpanSetList = []
2304  for span in maskSpanSetList:
2305  doKeep = True
2306  for footprint in footprintsToExclude.positive.getFootprints():
2307  if footprint.spans.contains(span):
2308  doKeep = False
2309  break
2310  if doKeep:
2311  filteredMaskSpanSetList.append(span)
2312  maskSpanSetList = filteredMaskSpanSetList
2313 
2314  return maskSpanSetList
2315 
2316  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2317  """Fetch a warp from the butler and return a warpDiff.
2318 
2319  Parameters
2320  ----------
2321  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2322  Butler dataRef for the warp.
2323  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2324  An image scaler object.
2325  templateCoadd : `lsst.afw.image.Exposure`
2326  Exposure to be substracted from the scaled warp.
2327 
2328  Returns
2329  -------
2330  warp : `lsst.afw.image.Exposure`
2331  Exposure of the image difference between the warp and template.
2332  """
2333 
2334  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2335  warpName = self.getTempExpDatasetName('psfMatched')
2336  if not warpRef.datasetExists(warpName):
2337  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2338  return None
2339  warp = warpRef.get(warpName, immediate=True)
2340  # direct image scaler OK for PSF-matched Warp
2341  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2342  mi = warp.getMaskedImage()
2343  if self.config.doScaleWarpVariance:
2344  try:
2345  self.scaleWarpVariance.run(mi)
2346  except Exception as exc:
2347  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2348  mi -= templateCoadd.getMaskedImage()
2349  return warp
2350 
2351  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2352  """Return a path to which to write debugging output.
2353 
2354  Creates a hyphen-delimited string of dataId values for simple filenames.
2355 
2356  Parameters
2357  ----------
2358  prefix : `str`
2359  Prefix for filename.
2360  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2361  Butler dataRef to make the path from.
2362  coaddLevel : `bool`, optional.
2363  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2364  'filter', but no 'visit').
2365 
2366  Returns
2367  -------
2368  result : `str`
2369  Path for debugging output.
2370  """
2371  if coaddLevel:
2372  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2373  else:
2374  keys = warpRef.dataId.keys()
2375  keyList = sorted(keys, reverse=True)
2376  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2377  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2378  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:149
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:100
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def getTempExpRefList(self, patchRef, calExpRefList)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:249
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:133
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:199
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:113
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def processResults(self, coaddExposure, dataRef)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)