lsst.pipe.tasks  17.0.1-19-g865eaf9b
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy
24 import warnings
25 import lsst.pex.config as pexConfig
26 import lsst.pex.exceptions as pexExceptions
27 import lsst.afw.geom as afwGeom
28 import lsst.afw.image as afwImage
29 import lsst.afw.math as afwMath
30 import lsst.afw.table as afwTable
31 import lsst.afw.detection as afwDet
32 import lsst.coadd.utils as coaddUtils
33 import lsst.pipe.base as pipeBase
34 import lsst.meas.algorithms as measAlg
35 import lsst.log as log
36 import lsstDebug
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo
38 from .interpImage import InterpImageTask
39 from .scaleZeroPoint import ScaleZeroPointTask
40 from .coaddHelpers import groupPatchExposures, getGroupDataRef
41 from .scaleVariance import ScaleVarianceTask
42 from lsst.meas.algorithms import SourceDetectionTask
43 from lsst.pipe.base.shims import ShimButler
44 
45 __all__ = ["AssembleCoaddTask", "AssembleCoaddConfig", "SafeClipAssembleCoaddTask",
46  "SafeClipAssembleCoaddConfig", "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
47 
48 
49 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig):
50  """Configuration parameters for the `AssembleCoaddTask`.
51 
52  Notes
53  -----
54  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
55  only set the bitplane config.brightObjectMaskName. To make this useful you
56  *must* also configure the flags.pixel algorithm, for example by adding
57 
58  .. code-block:: none
59 
60  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
61  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
62 
63  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
64  """
65  warpType = pexConfig.Field(
66  doc="Warp name: one of 'direct' or 'psfMatched'",
67  dtype=str,
68  default="direct",
69  )
70  subregionSize = pexConfig.ListField(
71  dtype=int,
72  doc="Width, height of stack subregion size; "
73  "make small enough that a full stack of images will fit into memory at once.",
74  length=2,
75  default=(2000, 2000),
76  )
77  statistic = pexConfig.Field(
78  dtype=str,
79  doc="Main stacking statistic for aggregating over the epochs.",
80  default="MEANCLIP",
81  )
82  doSigmaClip = pexConfig.Field(
83  dtype=bool,
84  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
85  default=False,
86  )
87  sigmaClip = pexConfig.Field(
88  dtype=float,
89  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
90  default=3.0,
91  )
92  clipIter = pexConfig.Field(
93  dtype=int,
94  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
95  default=2,
96  )
97  calcErrorFromInputVariance = pexConfig.Field(
98  dtype=bool,
99  doc="Calculate coadd variance from input variance by stacking statistic."
100  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
101  default=True,
102  )
103  scaleZeroPoint = pexConfig.ConfigurableField(
104  target=ScaleZeroPointTask,
105  doc="Task to adjust the photometric zero point of the coadd temp exposures",
106  )
107  doInterp = pexConfig.Field(
108  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
109  dtype=bool,
110  default=True,
111  )
112  interpImage = pexConfig.ConfigurableField(
113  target=InterpImageTask,
114  doc="Task to interpolate (and extrapolate) over NaN pixels",
115  )
116  doWrite = pexConfig.Field(
117  doc="Persist coadd?",
118  dtype=bool,
119  default=True,
120  )
121  doNImage = pexConfig.Field(
122  doc="Create image of number of contributing exposures for each pixel",
123  dtype=bool,
124  default=False,
125  )
126  doUsePsfMatchedPolygons = pexConfig.Field(
127  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
128  dtype=bool,
129  default=False,
130  )
131  maskPropagationThresholds = pexConfig.DictField(
132  keytype=str,
133  itemtype=float,
134  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
135  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
136  "would have contributed exceeds this value."),
137  default={"SAT": 0.1},
138  )
139  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
140  doc="Mask planes to remove before coadding")
141  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
142  doc="Set mask and flag bits for bright objects?")
143  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
144  doc="Name of mask bit used for bright objects")
145  coaddPsf = pexConfig.ConfigField(
146  doc="Configuration for CoaddPsf",
147  dtype=measAlg.CoaddPsfConfig,
148  )
149  doAttachTransmissionCurve = pexConfig.Field(
150  dtype=bool, default=False, optional=False,
151  doc=("Attach a piecewise TransmissionCurve for the coadd? "
152  "(requires all input Exposures to have TransmissionCurves).")
153  )
154  inputWarps = pipeBase.InputDatasetField(
155  doc=("Input list of warps to be assemebled i.e. stacked."
156  "WarpType (e.g. direct, psfMatched) is controlled by we warpType config parameter"),
157  nameTemplate="{inputCoaddName}Coadd_{warpType}Warp",
158  storageClass="ExposureF",
159  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
160  manualLoad=True,
161  )
162  skyMap = pipeBase.InputDatasetField(
163  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
164  nameTemplate="{inputCoaddName}Coadd_skyMap",
165  storageClass="SkyMap",
166  dimensions=("skymap", ),
167  scalar=True
168  )
169  brightObjectMask = pipeBase.InputDatasetField(
170  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
171  " BRIGHT_OBJECT."),
172  name="brightObjectMask",
173  storageClass="ObjectMaskCatalog",
174  dimensions=("tract", "patch", "skymap", "abstract_filter"),
175  scalar=True
176  )
177  coaddExposure = pipeBase.OutputDatasetField(
178  doc="Output coadded exposure, produced by stacking input warps",
179  nameTemplate="{outputCoaddName}Coadd",
180  storageClass="ExposureF",
181  dimensions=("tract", "patch", "skymap", "abstract_filter"),
182  scalar=True
183  )
184  nImage = pipeBase.OutputDatasetField(
185  doc="Output image of number of input images per pixel",
186  nameTemplate="{outputCoaddName}Coadd_nImage",
187  storageClass="ImageU",
188  dimensions=("tract", "patch", "skymap", "abstract_filter"),
189  scalar=True
190  )
191 
192  def setDefaults(self):
193  super().setDefaults()
194  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
195  self.formatTemplateNames({"inputCoaddName": "deep", "outputCoaddName": "deep",
196  "warpType": self.warpType})
197  self.quantum.dimensions = ("tract", "patch", "abstract_filter", "skymap")
198 
199  def validate(self):
200  super().validate()
201  if self.doPsfMatch:
202  # Backwards compatibility.
203  # Configs do not have loggers
204  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
205  self.warpType = 'psfMatched'
206  if self.doSigmaClip and self.statistic != "MEANCLIP":
207  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
208  self.statistic = "MEANCLIP"
209  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
210  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
211  "compute and set a non-zero coadd variance estimate." % (self.statistic))
212 
213  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
214  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
215  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
216  if str(k) not in unstackableStats]
217  raise ValueError("statistic %s is not allowed. Please choose one of %s."
218  % (self.statistic, stackableStats))
219 
220 
221 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
222  """Assemble a coadded image from a set of warps (coadded temporary exposures).
223 
224  We want to assemble a coadded image from a set of Warps (also called
225  coadded temporary exposures or ``coaddTempExps``).
226  Each input Warp covers a patch on the sky and corresponds to a single
227  run/visit/exposure of the covered patch. We provide the task with a list
228  of Warps (``selectDataList``) from which it selects Warps that cover the
229  specified patch (pointed at by ``dataRef``).
230  Each Warp that goes into a coadd will typically have an independent
231  photometric zero-point. Therefore, we must scale each Warp to set it to
232  a common photometric zeropoint. WarpType may be one of 'direct' or
233  'psfMatched', and the boolean configs `config.makeDirect` and
234  `config.makePsfMatched` set which of the warp types will be coadded.
235  The coadd is computed as a mean with optional outlier rejection.
236  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
237  Finally, Warps can have bad 'NaN' pixels which received no input from the
238  source calExps. We interpolate over these bad (NaN) pixels.
239 
240  `AssembleCoaddTask` uses several sub-tasks. These are
241 
242  - `ScaleZeroPointTask`
243  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
244  - `InterpImageTask`
245  - interpolate across bad pixels (NaN) in the final coadd
246 
247  You can retarget these subtasks if you wish.
248 
249  Notes
250  -----
251  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
252  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
253  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
254  no debug variables of its own. Some of the subtasks may support debug
255  variables. See the documentation for the subtasks for further information.
256 
257  Examples
258  --------
259  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
260  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
261  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
262  inputs: a data reference to the tract patch and filter to be coadded, and
263  a list of Warps to attempt to coadd. These are specified using ``--id`` and
264  ``--selectId``, respectively:
265 
266  .. code-block:: none
267 
268  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
269  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
270 
271  Only the Warps that cover the specified tract and patch will be coadded.
272  A list of the available optional arguments can be obtained by calling
273  ``assembleCoadd.py`` with the ``--help`` command line argument:
274 
275  .. code-block:: none
276 
277  assembleCoadd.py --help
278 
279  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
280  multi-band processing, we will generate the HSC-I & -R band coadds from
281  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
282  assuming that the lsst stack has been already set up, we must set up the
283  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
284  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
285  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
286  coadds, we must first
287 
288  - processCcd
289  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
290  - makeSkyMap
291  - create a skymap that covers the area of the sky present in the raw exposures
292  - makeCoaddTempExp
293  - warp the individual calibrated exposures to the tangent plane of the coadd
294 
295  We can perform all of these steps by running
296 
297  .. code-block:: none
298 
299  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
300 
301  This will produce warped exposures for each visit. To coadd the warped
302  data, we call assembleCoadd.py as follows:
303 
304  .. code-block:: none
305 
306  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
307  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
308  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
309  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
310  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
311  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
312  --selectId visit=903988 ccd=24
313 
314  that will process the HSC-I band data. The results are written in
315  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
316 
317  You may also choose to run:
318 
319  .. code-block:: none
320 
321  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
322  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
323  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
324  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
325  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
326  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
327  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
328  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
329 
330  to generate the coadd for the HSC-R band if you are interested in
331  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
332  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
333  rather than `AssembleCoaddTask` to make the coadd.
334  """
335  ConfigClass = AssembleCoaddConfig
336  _DefaultName = "assembleCoadd"
337 
338  def __init__(self, *args, **kwargs):
339  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
340  if args:
341  argNames = ["config", "name", "parentTask", "log"]
342  kwargs.update({k: v for k, v in zip(argNames, args)})
343  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
344  "PipelineTask will not take positional args" % argNames, FutureWarning)
345 
346  super().__init__(**kwargs)
347  self.makeSubtask("interpImage")
348  self.makeSubtask("scaleZeroPoint")
349 
350  if self.config.doMaskBrightObjects:
351  mask = afwImage.Mask()
352  try:
353  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
354  except pexExceptions.LsstCppException:
355  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
356  mask.getMaskPlaneDict().keys())
357  del mask
358 
359  self.warpType = self.config.warpType
360 
361  @classmethod
362  def getOutputDatasetTypes(cls, config):
363  """Return output dataset type descriptors
364 
365  Remove output dataset types not produced by the Task
366  """
367  outputTypeDict = super().getOutputDatasetTypes(config)
368  if not config.doNImage:
369  outputTypeDict.pop("nImage", None)
370  return outputTypeDict
371 
372  @classmethod
373  def getInputDatasetTypes(cls, config):
374  """Return input dataset type descriptors
375 
376  Remove input dataset types not used by the Task
377  """
378  inputTypeDict = super().getInputDatasetTypes(config)
379  if not config.doMaskBrightObjects:
380  inputTypeDict.pop("brightObjectMask", None)
381  return inputTypeDict
382 
383  @classmethod
384  def getPrerequisiteDatasetTypes(cls, config):
385  return frozenset(["brightObjectMask"])
386 
387  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
388  """Assemble a coadd from a set of Warps.
389 
390  PipelineTask (Gen3) entry point to Coadd a set of Warps.
391  Analogous to `runDataRef`, it prepares all the data products to be
392  passed to `run`, and processes the results before returning to struct
393  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
394  Therefore, its inputs are accessed subregion by subregion
395  by the `lsst.daf.butler.ShimButler` that quacks like a Gen2
396  `lsst.daf.persistence.Butler`. Updates to this method should
397  correspond to an update in `runDataRef` while both entry points
398  are used.
399 
400  Parameters
401  ----------
402  inputData : `dict`
403  Keys are the names of the configs describing input dataset types.
404  Values are input Python-domain data objects (or lists of objects)
405  retrieved from data butler.
406  inputDataIds : `dict`
407  Keys are the names of the configs describing input dataset types.
408  Values are DataIds (or lists of DataIds) that task consumes for
409  corresponding dataset type.
410  outputDataIds : `dict`
411  Keys are the names of the configs describing input dataset types.
412  Values are DataIds (or lists of DataIds) that task is to produce
413  for corresponding dataset type.
414  butler : `lsst.daf.butler.Butler`
415  Gen3 Butler object for fetching additional data products before
416  running the Task
417 
418  Returns
419  -------
420  result : `lsst.pipe.base.Struct`
421  Result struct with components:
422 
423  - ``coaddExposure`` : coadded exposure (``lsst.afw.image.Exposure``)
424  - ``nImage``: N Image (``lsst.afw.image.Image``)
425  """
426  # Construct skyInfo expected by run
427  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
428  skyMap = inputData["skyMap"]
429  outputDataId = next(iter(outputDataIds.values()))
430  inputData['skyInfo'] = makeSkyInfo(skyMap,
431  tractId=outputDataId['tract'],
432  patchId=outputDataId['patch'])
433 
434  # Construct list of input Shim DataRefs that quack like Gen2 DataRefs
435  butlerShim = ShimButler(butler)
436  warpRefList = [butlerShim.dataRef(self.config.inputWarps.name, dataId=dataId)
437  for dataId in inputDataIds['inputWarps']]
438 
439  # Construct output Shim DataRef
440  patchRef = butlerShim.dataRef(self.config.coaddExposure.name, dataId=outputDataIds['coaddExposure'])
441 
442  # Perform same middle steps as `runDataRef` does
443  inputs = self.prepareInputs(warpRefList)
444  self.log.info("Found %d %s", len(inputs.tempExpRefList),
445  self.getTempExpDatasetName(self.warpType))
446  if len(inputs.tempExpRefList) == 0:
447  self.log.warn("No coadd temporary exposures found")
448  return
449 
450  supplementaryData = self.makeSupplementaryDataGen3(inputData, inputDataIds, outputDataIds, butler)
451 
452  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
453  inputs.weightList, supplementaryData=supplementaryData)
454 
455  self.processResults(retStruct.coaddExposure, patchRef)
456  return retStruct
457 
458  @pipeBase.timeMethod
459  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
460  """Assemble a coadd from a set of Warps.
461 
462  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
463  Compute weights to be applied to each Warp and
464  find scalings to match the photometric zeropoint to a reference Warp.
465  Assemble the Warps using `run`. Interpolate over NaNs and
466  optionally write the coadd to disk. Return the coadded exposure.
467 
468  Parameters
469  ----------
470  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
471  Data reference defining the patch for coaddition and the
472  reference Warp (if ``config.autoReference=False``).
473  Used to access the following data products:
474  - ``self.config.coaddName + "Coadd_skyMap"``
475  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
476  - ``self.config.coaddName + "Coadd"``
477  selectDataList : `list`
478  List of data references to Calexps. Data to be coadded will be
479  selected from this list based on overlap with the patch defined
480  by dataRef, grouped by visit, and converted to a list of data
481  references to warps.
482  warpRefList : `list`
483  List of data references to Warps to be coadded.
484  Note: `warpRefList` is just the new name for `tempExpRefList`.
485 
486  Returns
487  -------
488  retStruct : `lsst.pipe.base.Struct`
489  Result struct with components:
490 
491  - ``coaddExposure``: coadded exposure (``Exposure``).
492  - ``nImage``: exposure count image (``Image``).
493  """
494  if selectDataList and warpRefList:
495  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
496  "and which to use is ambiguous. Please pass only one.")
497 
498  skyInfo = self.getSkyInfo(dataRef)
499  if warpRefList is None:
500  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
501  if len(calExpRefList) == 0:
502  self.log.warn("No exposures to coadd")
503  return
504  self.log.info("Coadding %d exposures", len(calExpRefList))
505 
506  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
507 
508  inputData = self.prepareInputs(warpRefList)
509  self.log.info("Found %d %s", len(inputData.tempExpRefList),
510  self.getTempExpDatasetName(self.warpType))
511  if len(inputData.tempExpRefList) == 0:
512  self.log.warn("No coadd temporary exposures found")
513  return
514 
515  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
516 
517  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
518  inputData.weightList, supplementaryData=supplementaryData)
519 
520  self.processResults(retStruct.coaddExposure, dataRef)
521  if self.config.doWrite:
522  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
523  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
524  if self.config.doNImage and retStruct.nImage is not None:
525  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
526 
527  return retStruct
528 
529  def processResults(self, coaddExposure, dataRef):
530  """Interpolate over missing data and mask bright stars.
531 
532  Parameters
533  ----------
534  coaddExposure : `lsst.afw.image.Exposure`
535  The coadded exposure to process.
536  dataRef : `lsst.daf.persistence.ButlerDataRef`
537  Butler data reference for supplementary data.
538  """
539  if self.config.doInterp:
540  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
541  # The variance must be positive; work around for DM-3201.
542  varArray = coaddExposure.variance.array
543  with numpy.errstate(invalid="ignore"):
544  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
545 
546  if self.config.doMaskBrightObjects:
547  brightObjectMasks = self.readBrightObjectMasks(dataRef)
548  self.setBrightObjectMasks(coaddExposure, dataRef.dataId, brightObjectMasks)
549 
550  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
551  """Make additional inputs to run() specific to subclasses (Gen2)
552 
553  Duplicates interface of `runDataRef` method
554  Available to be implemented by subclasses only if they need the
555  coadd dataRef for performing preliminary processing before
556  assembling the coadd.
557 
558  Parameters
559  ----------
560  dataRef : `lsst.daf.persistence.ButlerDataRef`
561  Butler data reference for supplementary data.
562  selectDataList : `list`
563  List of data references to Warps.
564  """
565  pass
566 
567  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
568  """Make additional inputs to run() specific to subclasses (Gen3)
569 
570  Duplicates interface of`adaptArgsAndRun` method.
571  Available to be implemented by subclasses only if they need the
572  coadd dataRef for performing preliminary processing before
573  assembling the coadd.
574 
575  Parameters
576  ----------
577  inputData : `dict`
578  Keys are the names of the configs describing input dataset types.
579  Values are input Python-domain data objects (or lists of objects)
580  retrieved from data butler.
581  inputDataIds : `dict`
582  Keys are the names of the configs describing input dataset types.
583  Values are DataIds (or lists of DataIds) that task consumes for
584  corresponding dataset type.
585  DataIds are guaranteed to match data objects in ``inputData``.
586  outputDataIds : `dict`
587  Keys are the names of the configs describing input dataset types.
588  Values are DataIds (or lists of DataIds) that task is to produce
589  for corresponding dataset type.
590  butler : `lsst.daf.butler.Butler`
591  Gen3 Butler object for fetching additional data products before
592  running the Task
593 
594  Returns
595  -------
596  result : `lsst.pipe.base.Struct`
597  Contains whatever additional data the subclass's `run` method needs
598  """
599  pass
600 
601  def getTempExpRefList(self, patchRef, calExpRefList):
602  """Generate list data references corresponding to warped exposures
603  that lie within the patch to be coadded.
604 
605  Parameters
606  ----------
607  patchRef : `dataRef`
608  Data reference for patch.
609  calExpRefList : `list`
610  List of data references for input calexps.
611 
612  Returns
613  -------
614  tempExpRefList : `list`
615  List of Warp/CoaddTempExp data references.
616  """
617  butler = patchRef.getButler()
618  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
619  self.getTempExpDatasetName(self.warpType))
620  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
621  g, groupData.keys) for
622  g in groupData.groups.keys()]
623  return tempExpRefList
624 
625  def prepareInputs(self, refList):
626  """Prepare the input warps for coaddition by measuring the weight for
627  each warp and the scaling for the photometric zero point.
628 
629  Each Warp has its own photometric zeropoint and background variance.
630  Before coadding these Warps together, compute a scale factor to
631  normalize the photometric zeropoint and compute the weight for each Warp.
632 
633  Parameters
634  ----------
635  refList : `list`
636  List of data references to tempExp
637 
638  Returns
639  -------
640  result : `lsst.pipe.base.Struct`
641  Result struct with components:
642 
643  - ``tempExprefList``: `list` of data references to tempExp.
644  - ``weightList``: `list` of weightings.
645  - ``imageScalerList``: `list` of image scalers.
646  """
647  statsCtrl = afwMath.StatisticsControl()
648  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
649  statsCtrl.setNumIter(self.config.clipIter)
650  statsCtrl.setAndMask(self.getBadPixelMask())
651  statsCtrl.setNanSafe(True)
652  # compute tempExpRefList: a list of tempExpRef that actually exist
653  # and weightList: a list of the weight of the associated coadd tempExp
654  # and imageScalerList: a list of scale factors for the associated coadd tempExp
655  tempExpRefList = []
656  weightList = []
657  imageScalerList = []
658  tempExpName = self.getTempExpDatasetName(self.warpType)
659  for tempExpRef in refList:
660  if not tempExpRef.datasetExists(tempExpName):
661  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
662  continue
663 
664  tempExp = tempExpRef.get(tempExpName, immediate=True)
665  # Ignore any input warp that is empty of data
666  if numpy.isnan(tempExp.image.array).all():
667  continue
668  maskedImage = tempExp.getMaskedImage()
669  imageScaler = self.scaleZeroPoint.computeImageScaler(
670  exposure=tempExp,
671  dataRef=tempExpRef,
672  )
673  try:
674  imageScaler.scaleMaskedImage(maskedImage)
675  except Exception as e:
676  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
677  continue
678  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
679  afwMath.MEANCLIP, statsCtrl)
680  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
681  weight = 1.0 / float(meanVar)
682  if not numpy.isfinite(weight):
683  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
684  continue
685  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
686 
687  del maskedImage
688  del tempExp
689 
690  tempExpRefList.append(tempExpRef)
691  weightList.append(weight)
692  imageScalerList.append(imageScaler)
693 
694  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
695  imageScalerList=imageScalerList)
696 
697  def prepareStats(self, mask=None):
698  """Prepare the statistics for coadding images.
699 
700  Parameters
701  ----------
702  mask : `int`, optional
703  Bit mask value to exclude from coaddition.
704 
705  Returns
706  -------
707  stats : `lsst.pipe.base.Struct`
708  Statistics structure with the following fields:
709 
710  - ``statsCtrl``: Statistics control object for coadd
711  (`lsst.afw.math.StatisticsControl`)
712  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
713  """
714  if mask is None:
715  mask = self.getBadPixelMask()
716  statsCtrl = afwMath.StatisticsControl()
717  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
718  statsCtrl.setNumIter(self.config.clipIter)
719  statsCtrl.setAndMask(mask)
720  statsCtrl.setNanSafe(True)
721  statsCtrl.setWeighted(True)
722  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
723  for plane, threshold in self.config.maskPropagationThresholds.items():
724  bit = afwImage.Mask.getMaskPlane(plane)
725  statsCtrl.setMaskPropagationThreshold(bit, threshold)
726  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
727  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
728 
729  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
730  altMaskList=None, mask=None, supplementaryData=None):
731  """Assemble a coadd from input warps
732 
733  Assemble the coadd using the provided list of coaddTempExps. Since
734  the full coadd covers a patch (a large area), the assembly is
735  performed over small areas on the image at a time in order to
736  conserve memory usage. Iterate over subregions within the outer
737  bbox of the patch using `assembleSubregion` to stack the corresponding
738  subregions from the coaddTempExps with the statistic specified.
739  Set the edge bits the coadd mask based on the weight map.
740 
741  Parameters
742  ----------
743  skyInfo : `lsst.pipe.base.Struct`
744  Struct with geometric information about the patch.
745  tempExpRefList : `list`
746  List of data references to Warps (previously called CoaddTempExps).
747  imageScalerList : `list`
748  List of image scalers.
749  weightList : `list`
750  List of weights
751  altMaskList : `list`, optional
752  List of alternate masks to use rather than those stored with
753  tempExp.
754  mask : `int`, optional
755  Bit mask value to exclude from coaddition.
756  supplementaryData : lsst.pipe.base.Struct, optional
757  Struct with additional data products needed to assemble coadd.
758  Only used by subclasses that implement `makeSupplementaryData`
759  and override `run`.
760 
761  Returns
762  -------
763  result : `lsst.pipe.base.Struct`
764  Result struct with components:
765 
766  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
767  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
768  """
769  tempExpName = self.getTempExpDatasetName(self.warpType)
770  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
771  stats = self.prepareStats(mask=mask)
772 
773  if altMaskList is None:
774  altMaskList = [None]*len(tempExpRefList)
775 
776  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
777  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
778  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
779  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
780  coaddMaskedImage = coaddExposure.getMaskedImage()
781  subregionSizeArr = self.config.subregionSize
782  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
783  # if nImage is requested, create a zero one which can be passed to assembleSubregion
784  if self.config.doNImage:
785  nImage = afwImage.ImageU(skyInfo.bbox)
786  else:
787  nImage = None
788  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
789  try:
790  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
791  weightList, altMaskList, stats.flags, stats.ctrl,
792  nImage=nImage)
793  except Exception as e:
794  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
795 
796  self.setInexactPsf(coaddMaskedImage.getMask())
797  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
798  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
799  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
800  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
801 
802  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
803  """Set the metadata for the coadd.
804 
805  This basic implementation sets the filter from the first input.
806 
807  Parameters
808  ----------
809  coaddExposure : `lsst.afw.image.Exposure`
810  The target exposure for the coadd.
811  tempExpRefList : `list`
812  List of data references to tempExp.
813  weightList : `list`
814  List of weights.
815  """
816  assert len(tempExpRefList) == len(weightList), "Length mismatch"
817  tempExpName = self.getTempExpDatasetName(self.warpType)
818  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
819  # (and we need more than just the PropertySet that contains the header), which is not possible
820  # with the current butler (see #2777).
821  tempExpList = [tempExpRef.get(tempExpName + "_sub",
822  bbox=afwGeom.Box2I(coaddExposure.getBBox().getMin(),
823  afwGeom.Extent2I(1, 1)), immediate=True)
824  for tempExpRef in tempExpRefList]
825  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
826 
827  coaddExposure.setFilter(tempExpList[0].getFilter())
828  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
829  coaddInputs.ccds.reserve(numCcds)
830  coaddInputs.visits.reserve(len(tempExpList))
831 
832  for tempExp, weight in zip(tempExpList, weightList):
833  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
834 
835  if self.config.doUsePsfMatchedPolygons:
836  self.shrinkValidPolygons(coaddInputs)
837 
838  coaddInputs.visits.sort()
839  if self.warpType == "psfMatched":
840  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
841  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
842  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
843  # having the maximum width (sufficient because square)
844  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
845  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
846  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
847  else:
848  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
849  self.config.coaddPsf.makeControl())
850  coaddExposure.setPsf(psf)
851  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
852  coaddExposure.getWcs())
853  coaddExposure.getInfo().setApCorrMap(apCorrMap)
854  if self.config.doAttachTransmissionCurve:
855  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
856  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
857 
858  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
859  altMaskList, statsFlags, statsCtrl, nImage=None):
860  """Assemble the coadd for a sub-region.
861 
862  For each coaddTempExp, check for (and swap in) an alternative mask
863  if one is passed. Remove mask planes listed in
864  `config.removeMaskPlanes`. Finally, stack the actual exposures using
865  `lsst.afw.math.statisticsStack` with the statistic specified by
866  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
867  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
868  an N-sigma clipped mean where N and iterations are specified by
869  statsCtrl. Assign the stacked subregion back to the coadd.
870 
871  Parameters
872  ----------
873  coaddExposure : `lsst.afw.image.Exposure`
874  The target exposure for the coadd.
875  bbox : `lsst.afw.geom.Box`
876  Sub-region to coadd.
877  tempExpRefList : `list`
878  List of data reference to tempExp.
879  imageScalerList : `list`
880  List of image scalers.
881  weightList : `list`
882  List of weights.
883  altMaskList : `list`
884  List of alternate masks to use rather than those stored with
885  tempExp, or None. Each element is dict with keys = mask plane
886  name to which to add the spans.
887  statsFlags : `lsst.afw.math.Property`
888  Property object for statistic for coadd.
889  statsCtrl : `lsst.afw.math.StatisticsControl`
890  Statistics control object for coadd.
891  nImage : `lsst.afw.image.ImageU`, optional
892  Keeps track of exposure count for each pixel.
893  """
894  self.log.debug("Computing coadd over %s", bbox)
895  tempExpName = self.getTempExpDatasetName(self.warpType)
896  coaddExposure.mask.addMaskPlane("REJECTED")
897  coaddExposure.mask.addMaskPlane("CLIPPED")
898  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
899  maskMap = self.setRejectedMaskMapping(statsCtrl)
900  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
901  maskedImageList = []
902  if nImage is not None:
903  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
904  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
905  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
906  maskedImage = exposure.getMaskedImage()
907  mask = maskedImage.getMask()
908  if altMask is not None:
909  self.applyAltMaskPlanes(mask, altMask)
910  imageScaler.scaleMaskedImage(maskedImage)
911 
912  # Add 1 for each pixel which is not excluded by the exclude mask.
913  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
914  if nImage is not None:
915  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
916  if self.config.removeMaskPlanes:
917  self.removeMaskPlanes(maskedImage)
918  maskedImageList.append(maskedImage)
919 
920  with self.timer("stack"):
921  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
922  clipped, # also set output to CLIPPED if sigma-clipped
923  maskMap)
924  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
925  if nImage is not None:
926  nImage.assign(subNImage, bbox)
927 
928  def removeMaskPlanes(self, maskedImage):
929  """Unset the mask of an image for mask planes specified in the config.
930 
931  Parameters
932  ----------
933  maskedImage : `lsst.afw.image.MaskedImage`
934  The masked image to be modified.
935  """
936  mask = maskedImage.getMask()
937  for maskPlane in self.config.removeMaskPlanes:
938  try:
939  mask &= ~mask.getPlaneBitMask(maskPlane)
940  except pexExceptions.InvalidParameterError:
941  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
942  maskPlane)
943 
944  @staticmethod
945  def setRejectedMaskMapping(statsCtrl):
946  """Map certain mask planes of the warps to new planes for the coadd.
947 
948  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
949  or CLIPPED, set it to REJECTED on the coadd.
950  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
951  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
952 
953  Parameters
954  ----------
955  statsCtrl : `lsst.afw.math.StatisticsControl`
956  Statistics control object for coadd
957 
958  Returns
959  -------
960  maskMap : `list` of `tuple` of `int`
961  A list of mappings of mask planes of the warped exposures to
962  mask planes of the coadd.
963  """
964  edge = afwImage.Mask.getPlaneBitMask("EDGE")
965  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
966  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
967  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
968  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
969  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
970  (clipped, clipped)]
971  return maskMap
972 
973  def applyAltMaskPlanes(self, mask, altMaskSpans):
974  """Apply in place alt mask formatted as SpanSets to a mask.
975 
976  Parameters
977  ----------
978  mask : `lsst.afw.image.Mask`
979  Original mask.
980  altMaskSpans : `dict`
981  SpanSet lists to apply. Each element contains the new mask
982  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
983  and list of SpanSets to apply to the mask.
984 
985  Returns
986  -------
987  mask : `lsst.afw.image.Mask`
988  Updated mask.
989  """
990  if self.config.doUsePsfMatchedPolygons:
991  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
992  # Clear away any other masks outside the validPolygons. These pixels are no longer
993  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
994  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
995  # This mask-clearing step must only occur *before* applying the new masks below
996  for spanSet in altMaskSpans['NO_DATA']:
997  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
998 
999  for plane, spanSetList in altMaskSpans.items():
1000  maskClipValue = mask.addMaskPlane(plane)
1001  for spanSet in spanSetList:
1002  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1003  return mask
1004 
1005  def shrinkValidPolygons(self, coaddInputs):
1006  """Shrink coaddInputs' ccds' ValidPolygons in place.
1007 
1008  Either modify each ccd's validPolygon in place, or if CoaddInputs
1009  does not have a validPolygon, create one from its bbox.
1010 
1011  Parameters
1012  ----------
1013  coaddInputs : `lsst.afw.image.coaddInputs`
1014  Original mask.
1015 
1016  """
1017  for ccd in coaddInputs.ccds:
1018  polyOrig = ccd.getValidPolygon()
1019  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1020  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1021  if polyOrig:
1022  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1023  else:
1024  validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
1025  ccd.setValidPolygon(validPolygon)
1026 
1027  def readBrightObjectMasks(self, dataRef):
1028  """Retrieve the bright object masks.
1029 
1030  Returns None on failure.
1031 
1032  Parameters
1033  ----------
1034  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1035  A Butler dataRef.
1036 
1037  Returns
1038  -------
1039  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1040  Bright object mask from the Butler object, or None if it cannot
1041  be retrieved.
1042  """
1043  try:
1044  return dataRef.get("brightObjectMask", immediate=True)
1045  except Exception as e:
1046  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1047  return None
1048 
1049  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
1050  """Set the bright object masks.
1051 
1052  Parameters
1053  ----------
1054  exposure : `lsst.afw.image.Exposure`
1055  Exposure under consideration.
1056  dataId : `lsst.daf.persistence.dataId`
1057  Data identifier dict for patch.
1058  brightObjectMasks : `lsst.afw.table`
1059  Table of bright objects to mask.
1060  """
1061 
1062  if brightObjectMasks is None:
1063  self.log.warn("Unable to apply bright object mask: none supplied")
1064  return
1065  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1066  mask = exposure.getMaskedImage().getMask()
1067  wcs = exposure.getWcs()
1068  plateScale = wcs.getPixelScale().asArcseconds()
1069 
1070  for rec in brightObjectMasks:
1071  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
1072  if rec["type"] == "box":
1073  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1074  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1075  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1076 
1077  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
1078  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
1079 
1080  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1081  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1082  spans = afwGeom.SpanSet(bbox)
1083  elif rec["type"] == "circle":
1084  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1085  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1086  else:
1087  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1088  continue
1089  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1090 
1091  def setInexactPsf(self, mask):
1092  """Set INEXACT_PSF mask plane.
1093 
1094  If any of the input images isn't represented in the coadd (due to
1095  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1096  these pixels.
1097 
1098  Parameters
1099  ----------
1100  mask : `lsst.afw.image.Mask`
1101  Coadded exposure's mask, modified in-place.
1102  """
1103  mask.addMaskPlane("INEXACT_PSF")
1104  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1105  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1106  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1107  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1108  array = mask.getArray()
1109  selected = array & (sensorEdge | clipped | rejected) > 0
1110  array[selected] |= inexactPsf
1111 
1112  @classmethod
1113  def _makeArgumentParser(cls):
1114  """Create an argument parser.
1115  """
1116  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1117  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
1118  cls.ConfigClass().warpType + "Warp",
1119  help="data ID, e.g. --id tract=12345 patch=1,2",
1120  ContainerClass=AssembleCoaddDataIdContainer)
1121  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1122  ContainerClass=SelectDataIdContainer)
1123  return parser
1124 
1125  @staticmethod
1126  def _subBBoxIter(bbox, subregionSize):
1127  """Iterate over subregions of a bbox.
1128 
1129  Parameters
1130  ----------
1131  bbox : `lsst.afw.geom.Box2I`
1132  Bounding box over which to iterate.
1133  subregionSize: `lsst.afw.geom.Extent2I`
1134  Size of sub-bboxes.
1135 
1136  Yields
1137  ------
1138  subBBox : `lsst.afw.geom.Box2I`
1139  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1140  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1141  the edges of ``bbox``, but it will never be empty.
1142  """
1143  if bbox.isEmpty():
1144  raise RuntimeError("bbox %s is empty" % (bbox,))
1145  if subregionSize[0] < 1 or subregionSize[1] < 1:
1146  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1147 
1148  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1149  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1150  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
1151  subBBox.clip(bbox)
1152  if subBBox.isEmpty():
1153  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1154  "colShift=%s, rowShift=%s" %
1155  (bbox, subregionSize, colShift, rowShift))
1156  yield subBBox
1157 
1158 
1159 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1160  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1161  """
1162 
1163  def makeDataRefList(self, namespace):
1164  """Make self.refList from self.idList.
1165 
1166  Parameters
1167  ----------
1168  namespace
1169  Results of parsing command-line (with ``butler`` and ``log`` elements).
1170  """
1171  datasetType = namespace.config.coaddName + "Coadd"
1172  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1173 
1174  for dataId in self.idList:
1175  # tract and patch are required
1176  for key in keysCoadd:
1177  if key not in dataId:
1178  raise RuntimeError("--id must include " + key)
1179 
1180  dataRef = namespace.butler.dataRef(
1181  datasetType=datasetType,
1182  dataId=dataId,
1183  )
1184  self.refList.append(dataRef)
1185 
1186 
1187 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1188  """Function to count the number of pixels with a specific mask in a
1189  footprint.
1190 
1191  Find the intersection of mask & footprint. Count all pixels in the mask
1192  that are in the intersection that have bitmask set but do not have
1193  ignoreMask set. Return the count.
1194 
1195  Parameters
1196  ----------
1197  mask : `lsst.afw.image.Mask`
1198  Mask to define intersection region by.
1199  footprint : `lsst.afw.detection.Footprint`
1200  Footprint to define the intersection region by.
1201  bitmask
1202  Specific mask that we wish to count the number of occurances of.
1203  ignoreMask
1204  Pixels to not consider.
1205 
1206  Returns
1207  -------
1208  result : `int`
1209  Count of number of pixels in footprint with specified mask.
1210  """
1211  bbox = footprint.getBBox()
1212  bbox.clip(mask.getBBox(afwImage.PARENT))
1213  fp = afwImage.Mask(bbox)
1214  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1215  footprint.spans.setMask(fp, bitmask)
1216  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1217  (subMask.getArray() & ignoreMask) == 0).sum()
1218 
1219 
1221  """Configuration parameters for the SafeClipAssembleCoaddTask.
1222  """
1223  clipDetection = pexConfig.ConfigurableField(
1224  target=SourceDetectionTask,
1225  doc="Detect sources on difference between unclipped and clipped coadd")
1226  minClipFootOverlap = pexConfig.Field(
1227  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1228  dtype=float,
1229  default=0.6
1230  )
1231  minClipFootOverlapSingle = pexConfig.Field(
1232  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1233  "clipped when only one visit overlaps",
1234  dtype=float,
1235  default=0.5
1236  )
1237  minClipFootOverlapDouble = pexConfig.Field(
1238  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1239  "clipped when two visits overlap",
1240  dtype=float,
1241  default=0.45
1242  )
1243  maxClipFootOverlapDouble = pexConfig.Field(
1244  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1245  "considering two visits",
1246  dtype=float,
1247  default=0.15
1248  )
1249  minBigOverlap = pexConfig.Field(
1250  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1251  "when labeling clipped footprints",
1252  dtype=int,
1253  default=100
1254  )
1255 
1256  def setDefaults(self):
1257  """Set default values for clipDetection.
1258 
1259  Notes
1260  -----
1261  The numeric values for these configuration parameters were
1262  empirically determined, future work may further refine them.
1263  """
1264  AssembleCoaddConfig.setDefaults(self)
1265  self.clipDetection.doTempLocalBackground = False
1266  self.clipDetection.reEstimateBackground = False
1267  self.clipDetection.returnOriginalFootprints = False
1268  self.clipDetection.thresholdPolarity = "both"
1269  self.clipDetection.thresholdValue = 2
1270  self.clipDetection.nSigmaToGrow = 2
1271  self.clipDetection.minPixels = 4
1272  self.clipDetection.isotropicGrow = True
1273  self.clipDetection.thresholdType = "pixel_stdev"
1274  self.sigmaClip = 1.5
1275  self.clipIter = 3
1276  self.statistic = "MEAN"
1277 
1278  def validate(self):
1279  if self.doSigmaClip:
1280  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1281  "Ignoring doSigmaClip.")
1282  self.doSigmaClip = False
1283  if self.statistic != "MEAN":
1284  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1285  "(%s chosen). Please set statistic to MEAN."
1286  % (self.statistic))
1287  AssembleCoaddTask.ConfigClass.validate(self)
1288 
1289 
1291  """Assemble a coadded image from a set of coadded temporary exposures,
1292  being careful to clip & flag areas with potential artifacts.
1293 
1294  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1295  we clip outliers). The problem with doing this is that when computing the
1296  coadd PSF at a given location, individual visit PSFs from visits with
1297  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1298  In this task, we correct for this behavior by creating a new
1299  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1300  coaddTempExps and the final coadd where
1301 
1302  i. difference imaging suggests that there is an outlier and
1303  ii. this outlier appears on only one or two images.
1304 
1305  Such regions will not contribute to the final coadd. Furthermore, any
1306  routine to determine the coadd PSF can now be cognizant of clipped regions.
1307  Note that the algorithm implemented by this task is preliminary and works
1308  correctly for HSC data. Parameter modifications and or considerable
1309  redesigning of the algorithm is likley required for other surveys.
1310 
1311  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1312  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1313  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1314  if you wish.
1315 
1316  Notes
1317  -----
1318  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1319  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1320  see `baseDebug` for more about ``debug.py`` files.
1321  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1322  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1323  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1324  for further information.
1325 
1326  Examples
1327  --------
1328  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1329  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1330  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1331 
1332  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1333  and filter to be coadded (specified using
1334  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1335  along with a list of coaddTempExps to attempt to coadd (specified using
1336  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1337  Only the coaddTempExps that cover the specified tract and patch will be
1338  coadded. A list of the available optional arguments can be obtained by
1339  calling assembleCoadd.py with the --help command line argument:
1340 
1341  .. code-block:: none
1342 
1343  assembleCoadd.py --help
1344 
1345  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1346  context of multi-band processing, we will generate the HSC-I & -R band
1347  coadds from HSC engineering test data provided in the ci_hsc package.
1348  To begin, assuming that the lsst stack has been already set up, we must
1349  set up the obs_subaru and ci_hsc packages. This defines the environment
1350  variable $CI_HSC_DIR and points at the location of the package. The raw
1351  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1352  the coadds, we must first
1353 
1354  - ``processCcd``
1355  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1356  - ``makeSkyMap``
1357  create a skymap that covers the area of the sky present in the raw exposures
1358  - ``makeCoaddTempExp``
1359  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1360 
1361  We can perform all of these steps by running
1362 
1363  .. code-block:: none
1364 
1365  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1366 
1367  This will produce warped coaddTempExps for each visit. To coadd the
1368  warped data, we call ``assembleCoadd.py`` as follows:
1369 
1370  .. code-block:: none
1371 
1372  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1373  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1374  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1375  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1376  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1377  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1378  --selectId visit=903988 ccd=24
1379 
1380  This will process the HSC-I band data. The results are written in
1381  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1382 
1383  You may also choose to run:
1384 
1385  .. code-block:: none
1386 
1387  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1388  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1389  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1390  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1391  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1392  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1393  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1394  --selectId visit=903346 ccd=12
1395 
1396  to generate the coadd for the HSC-R band if you are interested in following
1397  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1398  """
1399  ConfigClass = SafeClipAssembleCoaddConfig
1400  _DefaultName = "safeClipAssembleCoadd"
1401 
1402  def __init__(self, *args, **kwargs):
1403  AssembleCoaddTask.__init__(self, *args, **kwargs)
1404  schema = afwTable.SourceTable.makeMinimalSchema()
1405  self.makeSubtask("clipDetection", schema=schema)
1406 
1407  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1408  """Assemble the coadd for a region.
1409 
1410  Compute the difference of coadds created with and without outlier
1411  rejection to identify coadd pixels that have outlier values in some
1412  individual visits.
1413  Detect clipped regions on the difference image and mark these regions
1414  on the one or two individual coaddTempExps where they occur if there
1415  is significant overlap between the clipped region and a source. This
1416  leaves us with a set of footprints from the difference image that have
1417  been identified as having occured on just one or two individual visits.
1418  However, these footprints were generated from a difference image. It
1419  is conceivable for a large diffuse source to have become broken up
1420  into multiple footprints acrosss the coadd difference in this process.
1421  Determine the clipped region from all overlapping footprints from the
1422  detected sources in each visit - these are big footprints.
1423  Combine the small and big clipped footprints and mark them on a new
1424  bad mask plane.
1425  Generate the coadd using `AssembleCoaddTask.run` without outlier
1426  removal. Clipped footprints will no longer make it into the coadd
1427  because they are marked in the new bad mask plane.
1428 
1429  Parameters
1430  ----------
1431  skyInfo : `lsst.pipe.base.Struct`
1432  Patch geometry information, from getSkyInfo
1433  tempExpRefList : `list`
1434  List of data reference to tempExp
1435  imageScalerList : `list`
1436  List of image scalers
1437  weightList : `list`
1438  List of weights
1439 
1440  Returns
1441  -------
1442  result : `lsst.pipe.base.Struct`
1443  Result struct with components:
1444 
1445  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1446  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
1447 
1448  Notes
1449  -----
1450  args and kwargs are passed but ignored in order to match the call
1451  signature expected by the parent task.
1452  """
1453  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1454  mask = exp.getMaskedImage().getMask()
1455  mask.addMaskPlane("CLIPPED")
1456 
1457  result = self.detectClip(exp, tempExpRefList)
1458 
1459  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1460 
1461  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1462  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1463  # Append big footprints from individual Warps to result.clipSpans
1464  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1465  result.detectionFootprints, maskClipValue, maskDetValue,
1466  exp.getBBox())
1467  # Create mask of the current clipped footprints
1468  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1469  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1470 
1471  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1472  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1473  maskClip |= maskClipBig
1474 
1475  # Assemble coadd from base class, but ignoring CLIPPED pixels
1476  badMaskPlanes = self.config.badMaskPlanes[:]
1477  badMaskPlanes.append("CLIPPED")
1478  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1479  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1480  result.clipSpans, mask=badPixelMask)
1481 
1482  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1483  """Return an exposure that contains the difference between unclipped
1484  and clipped coadds.
1485 
1486  Generate a difference image between clipped and unclipped coadds.
1487  Compute the difference image by subtracting an outlier-clipped coadd
1488  from an outlier-unclipped coadd. Return the difference image.
1489 
1490  Parameters
1491  ----------
1492  skyInfo : `lsst.pipe.base.Struct`
1493  Patch geometry information, from getSkyInfo
1494  tempExpRefList : `list`
1495  List of data reference to tempExp
1496  imageScalerList : `list`
1497  List of image scalers
1498  weightList : `list`
1499  List of weights
1500 
1501  Returns
1502  -------
1503  exp : `lsst.afw.image.Exposure`
1504  Difference image of unclipped and clipped coadd wrapped in an Exposure
1505  """
1506  # Clone and upcast self.config because current self.config is frozen
1507  config = AssembleCoaddConfig()
1508  # getattr necessary because subtasks do not survive Config.toDict()
1509  configIntersection = {k: getattr(self.config, k)
1510  for k, v in self.config.toDict().items() if (k in config.keys())}
1511  config.update(**configIntersection)
1512 
1513  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1514  config.statistic = 'MEAN'
1515  task = AssembleCoaddTask(config=config)
1516  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1517 
1518  config.statistic = 'MEANCLIP'
1519  task = AssembleCoaddTask(config=config)
1520  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1521 
1522  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1523  coaddDiff -= coaddClip.getMaskedImage()
1524  exp = afwImage.ExposureF(coaddDiff)
1525  exp.setPsf(coaddMean.getPsf())
1526  return exp
1527 
1528  def detectClip(self, exp, tempExpRefList):
1529  """Detect clipped regions on an exposure and set the mask on the
1530  individual tempExp masks.
1531 
1532  Detect footprints in the difference image after smoothing the
1533  difference image with a Gaussian kernal. Identify footprints that
1534  overlap with one or two input ``coaddTempExps`` by comparing the
1535  computed overlap fraction to thresholds set in the config. A different
1536  threshold is applied depending on the number of overlapping visits
1537  (restricted to one or two). If the overlap exceeds the thresholds,
1538  the footprint is considered "CLIPPED" and is marked as such on the
1539  coaddTempExp. Return a struct with the clipped footprints, the indices
1540  of the ``coaddTempExps`` that end up overlapping with the clipped
1541  footprints, and a list of new masks for the ``coaddTempExps``.
1542 
1543  Parameters
1544  ----------
1545  exp : `lsst.afw.image.Exposure`
1546  Exposure to run detection on.
1547  tempExpRefList : `list`
1548  List of data reference to tempExp.
1549 
1550  Returns
1551  -------
1552  result : `lsst.pipe.base.Struct`
1553  Result struct with components:
1554 
1555  - ``clipFootprints``: list of clipped footprints.
1556  - ``clipIndices``: indices for each ``clippedFootprint`` in
1557  ``tempExpRefList``.
1558  - ``clipSpans``: List of dictionaries containing spanSet lists
1559  to clip. Each element contains the new maskplane name
1560  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1561  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1562  compressed into footprints.
1563  """
1564  mask = exp.getMaskedImage().getMask()
1565  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1566  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1567  # Merge positive and negative together footprints together
1568  fpSet.positive.merge(fpSet.negative)
1569  footprints = fpSet.positive
1570  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1571  ignoreMask = self.getBadPixelMask()
1572 
1573  clipFootprints = []
1574  clipIndices = []
1575  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1576 
1577  # for use by detectClipBig
1578  visitDetectionFootprints = []
1579 
1580  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1581  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1582  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1583 
1584  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1585  for i, warpRef in enumerate(tempExpRefList):
1586  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1587  immediate=True).getMaskedImage().getMask()
1588  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1589  afwImage.PARENT, True)
1590  maskVisitDet &= maskDetValue
1591  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1592  visitDetectionFootprints.append(visitFootprints)
1593 
1594  for j, footprint in enumerate(footprints.getFootprints()):
1595  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1596  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1597 
1598  # build a list of clipped spans for each visit
1599  for j, footprint in enumerate(footprints.getFootprints()):
1600  nPixel = footprint.getArea()
1601  overlap = [] # hold the overlap with each visit
1602  indexList = [] # index of visit in global list
1603  for i in range(len(tempExpRefList)):
1604  ignore = ignoreArr[i, j]
1605  overlapDet = overlapDetArr[i, j]
1606  totPixel = nPixel - ignore
1607 
1608  # If we have more bad pixels than detection skip
1609  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1610  continue
1611  overlap.append(overlapDet/float(totPixel))
1612  indexList.append(i)
1613 
1614  overlap = numpy.array(overlap)
1615  if not len(overlap):
1616  continue
1617 
1618  keep = False # Should this footprint be marked as clipped?
1619  keepIndex = [] # Which tempExps does the clipped footprint belong to
1620 
1621  # If footprint only has one overlap use a lower threshold
1622  if len(overlap) == 1:
1623  if overlap[0] > self.config.minClipFootOverlapSingle:
1624  keep = True
1625  keepIndex = [0]
1626  else:
1627  # This is the general case where only visit should be clipped
1628  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1629  if len(clipIndex) == 1:
1630  keep = True
1631  keepIndex = [clipIndex[0]]
1632 
1633  # Test if there are clipped objects that overlap two different visits
1634  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1635  if len(clipIndex) == 2 and len(overlap) > 3:
1636  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1637  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1638  keep = True
1639  keepIndex = clipIndex
1640 
1641  if not keep:
1642  continue
1643 
1644  for index in keepIndex:
1645  globalIndex = indexList[index]
1646  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1647 
1648  clipIndices.append(numpy.array(indexList)[keepIndex])
1649  clipFootprints.append(footprint)
1650 
1651  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1652  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1653 
1654  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1655  maskClipValue, maskDetValue, coaddBBox):
1656  """Return individual warp footprints for large artifacts and append
1657  them to ``clipList`` in place.
1658 
1659  Identify big footprints composed of many sources in the coadd
1660  difference that may have originated in a large diffuse source in the
1661  coadd. We do this by indentifying all clipped footprints that overlap
1662  significantly with each source in all the coaddTempExps.
1663 
1664  Parameters
1665  ----------
1666  clipList : `list`
1667  List of alt mask SpanSets with clipping information. Modified.
1668  clipFootprints : `list`
1669  List of clipped footprints.
1670  clipIndices : `list`
1671  List of which entries in tempExpClipList each footprint belongs to.
1672  maskClipValue
1673  Mask value of clipped pixels.
1674  maskDetValue
1675  Mask value of detected pixels.
1676  coaddBBox : `lsst.afw.geom.Box`
1677  BBox of the coadd and warps.
1678 
1679  Returns
1680  -------
1681  bigFootprintsCoadd : `list`
1682  List of big footprints
1683  """
1684  bigFootprintsCoadd = []
1685  ignoreMask = self.getBadPixelMask()
1686  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1687  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1688  for footprint in visitFootprints.getFootprints():
1689  footprint.spans.setMask(maskVisitDet, maskDetValue)
1690 
1691  # build a mask of clipped footprints that are in this visit
1692  clippedFootprintsVisit = []
1693  for foot, clipIndex in zip(clipFootprints, clipIndices):
1694  if index not in clipIndex:
1695  continue
1696  clippedFootprintsVisit.append(foot)
1697  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1698  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1699 
1700  bigFootprintsVisit = []
1701  for foot in visitFootprints.getFootprints():
1702  if foot.getArea() < self.config.minBigOverlap:
1703  continue
1704  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1705  if nCount > self.config.minBigOverlap:
1706  bigFootprintsVisit.append(foot)
1707  bigFootprintsCoadd.append(foot)
1708 
1709  for footprint in bigFootprintsVisit:
1710  clippedSpans["CLIPPED"].append(footprint.spans)
1711 
1712  return bigFootprintsCoadd
1713 
1714 
1716  assembleStaticSkyModel = pexConfig.ConfigurableField(
1717  target=AssembleCoaddTask,
1718  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1719  " naive/first-iteration model of the static sky.",
1720  )
1721  detect = pexConfig.ConfigurableField(
1722  target=SourceDetectionTask,
1723  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1724  )
1725  detectTemplate = pexConfig.ConfigurableField(
1726  target=SourceDetectionTask,
1727  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1728  )
1729  maxNumEpochs = pexConfig.Field(
1730  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1731  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1732  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1733  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1734  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1735  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1736  "than transient and not masked.",
1737  dtype=int,
1738  default=2
1739  )
1740  maxFractionEpochsLow = pexConfig.RangeField(
1741  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1742  "Effective maxNumEpochs = "
1743  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1744  dtype=float,
1745  default=0.4,
1746  min=0., max=1.,
1747  )
1748  maxFractionEpochsHigh = pexConfig.RangeField(
1749  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1750  "Effective maxNumEpochs = "
1751  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1752  dtype=float,
1753  default=0.03,
1754  min=0., max=1.,
1755  )
1756  spatialThreshold = pexConfig.RangeField(
1757  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1758  "temporal criteria. If 0, clip all. If 1, clip none.",
1759  dtype=float,
1760  default=0.5,
1761  min=0., max=1.,
1762  inclusiveMin=True, inclusiveMax=True
1763  )
1764  doScaleWarpVariance = pexConfig.Field(
1765  doc="Rescale Warp variance plane using empirical noise?",
1766  dtype=bool,
1767  default=True,
1768  )
1769  scaleWarpVariance = pexConfig.ConfigurableField(
1770  target=ScaleVarianceTask,
1771  doc="Rescale variance on warps",
1772  )
1773  doPreserveContainedBySource = pexConfig.Field(
1774  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1775  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1776  dtype=bool,
1777  default=True,
1778  )
1779  doPrefilterArtifacts = pexConfig.Field(
1780  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1781  "because they will be excluded anyway. This prevents them from contributing "
1782  "to the outlier epoch count image and potentially being labeled as persistant."
1783  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1784  dtype=bool,
1785  default=True
1786  )
1787  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1788  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1789  dtype=str,
1790  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1791  )
1792  prefilterArtifactsRatio = pexConfig.Field(
1793  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1794  dtype=float,
1795  default=0.05
1796  )
1797  psfMatchedWarps = pipeBase.InputDatasetField(
1798  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1799  "Only PSF-Matched Warps make sense for image subtraction. "
1800  "Therefore, they must be in the InputDatasetField and made available to the task."),
1801  nameTemplate="{inputCoaddName}Coadd_psfMatchedWarp",
1802  storageClass="ExposureF",
1803  dimensions=("tract", "patch", "skymap", "visit"),
1804  manualLoad=True
1805  )
1806 
1807  def setDefaults(self):
1808  AssembleCoaddConfig.setDefaults(self)
1809  self.statistic = 'MEAN'
1811 
1812  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1813  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1814  if "EDGE" in self.badMaskPlanes:
1815  self.badMaskPlanes.remove('EDGE')
1816  self.removeMaskPlanes.append('EDGE')
1817  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1818  self.assembleStaticSkyModel.warpType = 'psfMatched'
1819  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1820  self.assembleStaticSkyModel.sigmaClip = 2.5
1821  self.assembleStaticSkyModel.clipIter = 3
1822  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1823  self.assembleStaticSkyModel.doWrite = False
1824  self.detect.doTempLocalBackground = False
1825  self.detect.reEstimateBackground = False
1826  self.detect.returnOriginalFootprints = False
1827  self.detect.thresholdPolarity = "both"
1828  self.detect.thresholdValue = 5
1829  self.detect.nSigmaToGrow = 2
1830  self.detect.minPixels = 4
1831  self.detect.isotropicGrow = True
1832  self.detect.thresholdType = "pixel_stdev"
1833  self.detectTemplate.nSigmaToGrow = 2.0
1834  self.detectTemplate.doTempLocalBackground = False
1835  self.detectTemplate.reEstimateBackground = False
1836  self.detectTemplate.returnOriginalFootprints = False
1837 
1838 
1840  """Assemble a compareWarp coadded image from a set of warps
1841  by masking artifacts detected by comparing PSF-matched warps.
1842 
1843  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1844  we clip outliers). The problem with doing this is that when computing the
1845  coadd PSF at a given location, individual visit PSFs from visits with
1846  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1847  In this task, we correct for this behavior by creating a new badMaskPlane
1848  'CLIPPED' which marks pixels in the individual warps suspected to contain
1849  an artifact. We populate this plane on the input warps by comparing
1850  PSF-matched warps with a PSF-matched median coadd which serves as a
1851  model of the static sky. Any group of pixels that deviates from the
1852  PSF-matched template coadd by more than config.detect.threshold sigma,
1853  is an artifact candidate. The candidates are then filtered to remove
1854  variable sources and sources that are difficult to subtract such as
1855  bright stars. This filter is configured using the config parameters
1856  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1857  the maximum fraction of epochs that the deviation can appear in and still
1858  be considered an artifact. The spatialThreshold is the maximum fraction of
1859  pixels in the footprint of the deviation that appear in other epochs
1860  (where other epochs is defined by the temporalThreshold). If the deviant
1861  region meets this criteria of having a significant percentage of pixels
1862  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1863  set in the mask. These regions will not contribute to the final coadd.
1864  Furthermore, any routine to determine the coadd PSF can now be cognizant
1865  of clipped regions. Note that the algorithm implemented by this task is
1866  preliminary and works correctly for HSC data. Parameter modifications and
1867  or considerable redesigning of the algorithm is likley required for other
1868  surveys.
1869 
1870  ``CompareWarpAssembleCoaddTask`` sub-classes
1871  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1872  as a subtask to generate the TemplateCoadd (the model of the static sky).
1873 
1874  Notes
1875  -----
1876  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1877  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1878  ``baseDebug`` for more about ``debug.py`` files.
1879 
1880  This task supports the following debug variables:
1881 
1882  - ``saveCountIm``
1883  If True then save the Epoch Count Image as a fits file in the `figPath`
1884  - ``figPath``
1885  Path to save the debug fits images and figures
1886 
1887  For example, put something like:
1888 
1889  .. code-block:: python
1890 
1891  import lsstDebug
1892  def DebugInfo(name):
1893  di = lsstDebug.getInfo(name)
1894  if name == "lsst.pipe.tasks.assembleCoadd":
1895  di.saveCountIm = True
1896  di.figPath = "/desired/path/to/debugging/output/images"
1897  return di
1898  lsstDebug.Info = DebugInfo
1899 
1900  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1901  ``--debug`` flag. Some subtasks may have their own debug variables;
1902  see individual Task documentation.
1903 
1904  Examples
1905  --------
1906  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1907  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1908  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1909  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1910  and filter to be coadded (specified using
1911  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1912  along with a list of coaddTempExps to attempt to coadd (specified using
1913  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1914  Only the warps that cover the specified tract and patch will be coadded.
1915  A list of the available optional arguments can be obtained by calling
1916  ``assembleCoadd.py`` with the ``--help`` command line argument:
1917 
1918  .. code-block:: none
1919 
1920  assembleCoadd.py --help
1921 
1922  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1923  context of multi-band processing, we will generate the HSC-I & -R band
1924  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1925  To begin, assuming that the lsst stack has been already set up, we must
1926  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1927  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1928  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1929  directory. To begin assembling the coadds, we must first
1930 
1931  - processCcd
1932  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1933  - makeSkyMap
1934  create a skymap that covers the area of the sky present in the raw exposures
1935  - makeCoaddTempExp
1936  warp the individual calibrated exposures to the tangent plane of the coadd
1937 
1938  We can perform all of these steps by running
1939 
1940  .. code-block:: none
1941 
1942  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1943 
1944  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1945  warped data, we call ``assembleCoadd.py`` as follows:
1946 
1947  .. code-block:: none
1948 
1949  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1950  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1951  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1952  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1953  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1954  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1955  --selectId visit=903988 ccd=24
1956 
1957  This will process the HSC-I band data. The results are written in
1958  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1959  """
1960  ConfigClass = CompareWarpAssembleCoaddConfig
1961  _DefaultName = "compareWarpAssembleCoadd"
1962 
1963  def __init__(self, *args, **kwargs):
1964  AssembleCoaddTask.__init__(self, *args, **kwargs)
1965  self.makeSubtask("assembleStaticSkyModel")
1966  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1967  self.makeSubtask("detect", schema=detectionSchema)
1968  if self.config.doPreserveContainedBySource:
1969  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1970  if self.config.doScaleWarpVariance:
1971  self.makeSubtask("scaleWarpVariance")
1972 
1973  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
1974  """Make inputs specific to Subclass with Gen 3 API
1975 
1976  Calls Gen3 `adaptArgsAndRun` instead of the Gen2 specific `runDataRef`
1977 
1978  Duplicates interface of`adaptArgsAndRun` method.
1979  Available to be implemented by subclasses only if they need the
1980  coadd dataRef for performing preliminary processing before
1981  assembling the coadd.
1982 
1983  Parameters
1984  ----------
1985  inputData : `dict`
1986  Keys are the names of the configs describing input dataset types.
1987  Values are input Python-domain data objects (or lists of objects)
1988  retrieved from data butler.
1989  inputDataIds : `dict`
1990  Keys are the names of the configs describing input dataset types.
1991  Values are DataIds (or lists of DataIds) that task consumes for
1992  corresponding dataset type.
1993  DataIds are guaranteed to match data objects in ``inputData``.
1994  outputDataIds : `dict`
1995  Keys are the names of the configs describing input dataset types.
1996  Values are DataIds (or lists of DataIds) that task is to produce
1997  for corresponding dataset type.
1998  butler : `lsst.daf.butler.Butler`
1999  Gen3 Butler object for fetching additional data products before
2000  running the Task
2001 
2002  Returns
2003  -------
2004  result : `lsst.pipe.base.Struct`
2005  Result struct with components:
2006 
2007  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2008  - ``nImage``: N Image (``lsst.afw.image.Image``)
2009 
2010 
2011  """
2012  templateCoadd = self.assembleStaticSkyModel.adaptArgsAndRun(inputData, inputDataIds,
2013  outputDataIds, butler)
2014  if templateCoadd is None:
2015  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2016 
2017  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2018  nImage=templateCoadd.nImage)
2019 
2020  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2021  """Make inputs specific to Subclass.
2022 
2023  Generate a templateCoadd to use as a native model of static sky to
2024  subtract from warps.
2025 
2026  Parameters
2027  ----------
2028  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2029  Butler dataRef for supplementary data.
2030  selectDataList : `list` (optional)
2031  Optional List of data references to Calexps.
2032  warpRefList : `list` (optional)
2033  Optional List of data references to Warps.
2034 
2035  Returns
2036  -------
2037  result : `lsst.pipe.base.Struct`
2038  Result struct with components:
2039 
2040  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2041  - ``nImage``: N Image (``lsst.afw.image.Image``)
2042  """
2043  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2044  if templateCoadd is None:
2045  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2046 
2047  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2048  nImage=templateCoadd.nImage)
2049 
2050  def _noTemplateMessage(self, warpType):
2051  warpName = (warpType[0].upper() + warpType[1:])
2052  message = """No %(warpName)s warps were found to build the template coadd which is
2053  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2054  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2055  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2056 
2057  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2058  another algorithm like:
2059 
2060  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2061  config.assemble.retarget(SafeClipAssembleCoaddTask)
2062  """ % {"warpName": warpName}
2063  return message
2064 
2065  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2066  supplementaryData, *args, **kwargs):
2067  """Assemble the coadd.
2068 
2069  Find artifacts and apply them to the warps' masks creating a list of
2070  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2071  plane. Then pass these alternative masks to the base class's `run`
2072  method.
2073 
2074  Parameters
2075  ----------
2076  skyInfo : `lsst.pipe.base.Struct`
2077  Patch geometry information.
2078  tempExpRefList : `list`
2079  List of data references to warps.
2080  imageScalerList : `list`
2081  List of image scalers.
2082  weightList : `list`
2083  List of weights.
2084  supplementaryData : `lsst.pipe.base.Struct`
2085  This Struct must contain a ``templateCoadd`` that serves as the
2086  model of the static sky.
2087 
2088  Returns
2089  -------
2090  result : `lsst.pipe.base.Struct`
2091  Result struct with components:
2092 
2093  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
2094  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
2095  """
2096  templateCoadd = supplementaryData.templateCoadd
2097  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
2098  badMaskPlanes = self.config.badMaskPlanes[:]
2099  badMaskPlanes.append("CLIPPED")
2100  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2101 
2102  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2103  spanSetMaskList, mask=badPixelMask)
2104 
2105  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2106  # Psf-Matching moves the real edge inwards
2107  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2108  return result
2109 
2110  def applyAltEdgeMask(self, mask, altMaskList):
2111  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2112 
2113  Parameters
2114  ----------
2115  mask : `lsst.afw.image.Mask`
2116  Original mask.
2117  altMaskList : `list`
2118  List of Dicts containing ``spanSet`` lists.
2119  Each element contains the new mask plane name (e.g. "CLIPPED
2120  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2121  the mask.
2122  """
2123  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2124  for visitMask in altMaskList:
2125  if "EDGE" in visitMask:
2126  for spanSet in visitMask['EDGE']:
2127  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2128 
2129  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2130  """Find artifacts.
2131 
2132  Loop through warps twice. The first loop builds a map with the count
2133  of how many epochs each pixel deviates from the templateCoadd by more
2134  than ``config.chiThreshold`` sigma. The second loop takes each
2135  difference image and filters the artifacts detected in each using
2136  count map to filter out variable sources and sources that are
2137  difficult to subtract cleanly.
2138 
2139  Parameters
2140  ----------
2141  templateCoadd : `lsst.afw.image.Exposure`
2142  Exposure to serve as model of static sky.
2143  tempExpRefList : `list`
2144  List of data references to warps.
2145  imageScalerList : `list`
2146  List of image scalers.
2147 
2148  Returns
2149  -------
2150  altMasks : `list`
2151  List of dicts containing information about CLIPPED
2152  (i.e., artifacts), NO_DATA, and EDGE pixels.
2153  """
2154 
2155  self.log.debug("Generating Count Image, and mask lists.")
2156  coaddBBox = templateCoadd.getBBox()
2157  slateIm = afwImage.ImageU(coaddBBox)
2158  epochCountImage = afwImage.ImageU(coaddBBox)
2159  nImage = afwImage.ImageU(coaddBBox)
2160  spanSetArtifactList = []
2161  spanSetNoDataMaskList = []
2162  spanSetEdgeList = []
2163  badPixelMask = self.getBadPixelMask()
2164 
2165  # mask of the warp diffs should = that of only the warp
2166  templateCoadd.mask.clearAllMaskPlanes()
2167 
2168  if self.config.doPreserveContainedBySource:
2169  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2170  else:
2171  templateFootprints = None
2172 
2173  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2174  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2175  if warpDiffExp is not None:
2176  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2177  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2178  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2179  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2180  fpSet.positive.merge(fpSet.negative)
2181  footprints = fpSet.positive
2182  slateIm.set(0)
2183  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2184 
2185  # Remove artifacts due to defects before they contribute to the epochCountImage
2186  if self.config.doPrefilterArtifacts:
2187  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2188  for spans in spanSetList:
2189  spans.setImage(slateIm, 1, doClip=True)
2190  epochCountImage += slateIm
2191 
2192  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2193  # undergo a second convolution. Pixels with data in the direct warp
2194  # but not in the PSF-matched warp will not have their artifacts detected.
2195  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2196  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2197  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2198  nansMask.setXY0(warpDiffExp.getXY0())
2199  edgeMask = warpDiffExp.mask
2200  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2201  edgeMask.getPlaneBitMask("EDGE")).split()
2202  else:
2203  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2204  # In this case, mask the whole epoch
2205  nansMask = afwImage.MaskX(coaddBBox, 1)
2206  spanSetList = []
2207  spanSetEdgeMask = []
2208 
2209  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2210 
2211  spanSetNoDataMaskList.append(spanSetNoDataMask)
2212  spanSetArtifactList.append(spanSetList)
2213  spanSetEdgeList.append(spanSetEdgeMask)
2214 
2215  if lsstDebug.Info(__name__).saveCountIm:
2216  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2217  epochCountImage.writeFits(path)
2218 
2219  for i, spanSetList in enumerate(spanSetArtifactList):
2220  if spanSetList:
2221  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2222  templateFootprints)
2223  spanSetArtifactList[i] = filteredSpanSetList
2224 
2225  altMasks = []
2226  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2227  altMasks.append({'CLIPPED': artifacts,
2228  'NO_DATA': noData,
2229  'EDGE': edge})
2230  return altMasks
2231 
2232  def prefilterArtifacts(self, spanSetList, exp):
2233  """Remove artifact candidates covered by bad mask plane.
2234 
2235  Any future editing of the candidate list that does not depend on
2236  temporal information should go in this method.
2237 
2238  Parameters
2239  ----------
2240  spanSetList : `list`
2241  List of SpanSets representing artifact candidates.
2242  exp : `lsst.afw.image.Exposure`
2243  Exposure containing mask planes used to prefilter.
2244 
2245  Returns
2246  -------
2247  returnSpanSetList : `list`
2248  List of SpanSets with artifacts.
2249  """
2250  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2251  goodArr = (exp.mask.array & badPixelMask) == 0
2252  returnSpanSetList = []
2253  bbox = exp.getBBox()
2254  x0, y0 = exp.getXY0()
2255  for i, span in enumerate(spanSetList):
2256  y, x = span.clippedTo(bbox).indices()
2257  yIndexLocal = numpy.array(y) - y0
2258  xIndexLocal = numpy.array(x) - x0
2259  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2260  if goodRatio > self.config.prefilterArtifactsRatio:
2261  returnSpanSetList.append(span)
2262  return returnSpanSetList
2263 
2264  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2265  """Filter artifact candidates.
2266 
2267  Parameters
2268  ----------
2269  spanSetList : `list`
2270  List of SpanSets representing artifact candidates.
2271  epochCountImage : `lsst.afw.image.Image`
2272  Image of accumulated number of warpDiff detections.
2273  nImage : `lsst.afw.image.Image`
2274  Image of the accumulated number of total epochs contributing.
2275 
2276  Returns
2277  -------
2278  maskSpanSetList : `list`
2279  List of SpanSets with artifacts.
2280  """
2281 
2282  maskSpanSetList = []
2283  x0, y0 = epochCountImage.getXY0()
2284  for i, span in enumerate(spanSetList):
2285  y, x = span.indices()
2286  yIdxLocal = [y1 - y0 for y1 in y]
2287  xIdxLocal = [x1 - x0 for x1 in x]
2288  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2289  totalN = nImage.array[yIdxLocal, xIdxLocal]
2290 
2291  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2292  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2293  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2294  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2295  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2296  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2297  (outlierN <= effectiveMaxNumEpochs))
2298  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2299  if percentBelowThreshold > self.config.spatialThreshold:
2300  maskSpanSetList.append(span)
2301 
2302  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2303  # If a candidate is contained by a footprint on the template coadd, do not clip
2304  filteredMaskSpanSetList = []
2305  for span in maskSpanSetList:
2306  doKeep = True
2307  for footprint in footprintsToExclude.positive.getFootprints():
2308  if footprint.spans.contains(span):
2309  doKeep = False
2310  break
2311  if doKeep:
2312  filteredMaskSpanSetList.append(span)
2313  maskSpanSetList = filteredMaskSpanSetList
2314 
2315  return maskSpanSetList
2316 
2317  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2318  """Fetch a warp from the butler and return a warpDiff.
2319 
2320  Parameters
2321  ----------
2322  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2323  Butler dataRef for the warp.
2324  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2325  An image scaler object.
2326  templateCoadd : `lsst.afw.image.Exposure`
2327  Exposure to be substracted from the scaled warp.
2328 
2329  Returns
2330  -------
2331  warp : `lsst.afw.image.Exposure`
2332  Exposure of the image difference between the warp and template.
2333  """
2334 
2335  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2336  warpName = self.getTempExpDatasetName('psfMatched')
2337  if not warpRef.datasetExists(warpName):
2338  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2339  return None
2340  warp = warpRef.get(warpName, immediate=True)
2341  # direct image scaler OK for PSF-matched Warp
2342  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2343  mi = warp.getMaskedImage()
2344  if self.config.doScaleWarpVariance:
2345  try:
2346  self.scaleWarpVariance.run(mi)
2347  except Exception as exc:
2348  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2349  mi -= templateCoadd.getMaskedImage()
2350  return warp
2351 
2352  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2353  """Return a path to which to write debugging output.
2354 
2355  Creates a hyphen-delimited string of dataId values for simple filenames.
2356 
2357  Parameters
2358  ----------
2359  prefix : `str`
2360  Prefix for filename.
2361  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2362  Butler dataRef to make the path from.
2363  coaddLevel : `bool`, optional.
2364  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2365  'filter', but no 'visit').
2366 
2367  Returns
2368  -------
2369  result : `str`
2370  Path for debugging output.
2371  """
2372  if coaddLevel:
2373  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2374  else:
2375  keys = warpRef.dataId.keys()
2376  keyList = sorted(keys, reverse=True)
2377  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2378  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2379  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:149
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:100
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def getTempExpRefList(self, patchRef, calExpRefList)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:249
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:133
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:199
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:113
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def processResults(self, coaddExposure, dataRef)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)