lsst.pipe.tasks  16.0-61-gb2b2650a+2
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy
24 import warnings
25 import lsst.pex.config as pexConfig
26 import lsst.pex.exceptions as pexExceptions
27 import lsst.afw.geom as afwGeom
28 import lsst.afw.image as afwImage
29 import lsst.afw.math as afwMath
30 import lsst.afw.table as afwTable
31 import lsst.afw.detection as afwDet
32 import lsst.coadd.utils as coaddUtils
33 import lsst.pipe.base as pipeBase
34 import lsst.meas.algorithms as measAlg
35 import lsst.log as log
36 import lsstDebug
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo
38 from .interpImage import InterpImageTask
39 from .scaleZeroPoint import ScaleZeroPointTask
40 from .coaddHelpers import groupPatchExposures, getGroupDataRef
41 from .scaleVariance import ScaleVarianceTask
42 from lsst.meas.algorithms import SourceDetectionTask
43 from lsst.pipe.base.shims import ShimButler
44 
45 __all__ = ["AssembleCoaddTask", "AssembleCoaddConfig", "SafeClipAssembleCoaddTask",
46  "SafeClipAssembleCoaddConfig", "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
47 
48 
49 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig):
50  """Configuration parameters for the `AssembleCoaddTask`.
51 
52  Notes
53  -----
54  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
55  only set the bitplane config.brightObjectMaskName. To make this useful you
56  *must* also configure the flags.pixel algorithm, for example by adding
57 
58  .. code-block:: none
59 
60  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
61  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
62 
63  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
64  """
65  warpType = pexConfig.Field(
66  doc="Warp name: one of 'direct' or 'psfMatched'",
67  dtype=str,
68  default="direct",
69  )
70  subregionSize = pexConfig.ListField(
71  dtype=int,
72  doc="Width, height of stack subregion size; "
73  "make small enough that a full stack of images will fit into memory at once.",
74  length=2,
75  default=(2000, 2000),
76  )
77  statistic = pexConfig.Field(
78  dtype=str,
79  doc="Main stacking statistic for aggregating over the epochs.",
80  default="MEANCLIP",
81  )
82  doSigmaClip = pexConfig.Field(
83  dtype=bool,
84  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
85  default=False,
86  )
87  sigmaClip = pexConfig.Field(
88  dtype=float,
89  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
90  default=3.0,
91  )
92  clipIter = pexConfig.Field(
93  dtype=int,
94  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
95  default=2,
96  )
97  calcErrorFromInputVariance = pexConfig.Field(
98  dtype=bool,
99  doc="Calculate coadd variance from input variance by stacking statistic."
100  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
101  default=True,
102  )
103  scaleZeroPoint = pexConfig.ConfigurableField(
104  target=ScaleZeroPointTask,
105  doc="Task to adjust the photometric zero point of the coadd temp exposures",
106  )
107  doInterp = pexConfig.Field(
108  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
109  dtype=bool,
110  default=True,
111  )
112  interpImage = pexConfig.ConfigurableField(
113  target=InterpImageTask,
114  doc="Task to interpolate (and extrapolate) over NaN pixels",
115  )
116  doWrite = pexConfig.Field(
117  doc="Persist coadd?",
118  dtype=bool,
119  default=True,
120  )
121  doNImage = pexConfig.Field(
122  doc="Create image of number of contributing exposures for each pixel",
123  dtype=bool,
124  default=False,
125  )
126  doUsePsfMatchedPolygons = pexConfig.Field(
127  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
128  dtype=bool,
129  default=False,
130  )
131  maskPropagationThresholds = pexConfig.DictField(
132  keytype=str,
133  itemtype=float,
134  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
135  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
136  "would have contributed exceeds this value."),
137  default={"SAT": 0.1},
138  )
139  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
140  doc="Mask planes to remove before coadding")
141  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
142  doc="Set mask and flag bits for bright objects?")
143  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
144  doc="Name of mask bit used for bright objects")
145  coaddPsf = pexConfig.ConfigField(
146  doc="Configuration for CoaddPsf",
147  dtype=measAlg.CoaddPsfConfig,
148  )
149  doAttachTransmissionCurve = pexConfig.Field(
150  dtype=bool, default=False, optional=False,
151  doc=("Attach a piecewise TransmissionCurve for the coadd? "
152  "(requires all input Exposures to have TransmissionCurves).")
153  )
154  inputWarps = pipeBase.InputDatasetField(
155  doc=("Input list of warps to be assemebled i.e. stacked."
156  "WarpType (e.g. direct, psfMatched) is controlled by we warpType config parameter"),
157  nameTemplate="{inputCoaddName}Coadd_{warpType}Warp",
158  storageClass="ExposureF",
159  dimensions=("Tract", "Patch", "SkyMap", "Visit"),
160  manualLoad=True,
161  )
162  skyMap = pipeBase.InputDatasetField(
163  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
164  nameTemplate="{inputCoaddName}Coadd_skyMap",
165  storageClass="SkyMap",
166  dimensions=("SkyMap", ),
167  scalar=True
168  )
169  brightObjectMask = pipeBase.InputDatasetField(
170  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
171  "BRIGHT_OBJECT. Currently unavailable in Gen3 test repos. TODO: DM-17300"),
172  name="brightObjectMask",
173  storageClass="", # TODO: Change per storage chosen in DM-17300
174  dimensions=("Tract", "Patch", "SkyMap", "AbstractFilter"),
175  scalar=True
176  )
177  coaddExposure = pipeBase.OutputDatasetField(
178  doc="Output coadded exposure, produced by stacking input warps",
179  nameTemplate="{outputCoaddName}Coadd",
180  storageClass="ExposureF",
181  dimensions=("Tract", "Patch", "SkyMap", "AbstractFilter"),
182  scalar=True
183  )
184  nImage = pipeBase.OutputDatasetField(
185  doc="Output image of number of input images per pixel",
186  nameTemplate="{outputCoaddName}Coadd_nImage",
187  storageClass="ImageU",
188  dimensions=("Tract", "Patch", "SkyMap", "AbstractFilter"),
189  scalar=True
190  )
191 
192  def setDefaults(self):
193  super().setDefaults()
194  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
195  self.formatTemplateNames({"inputCoaddName": "deep", "outputCoaddName": "deep",
196  "warpType": self.warpType})
197  self.quantum.dimensions = ("Tract", "Patch", "AbstractFilter", "SkyMap")
198 
199  def validate(self):
200  super().validate()
201  if self.doPsfMatch:
202  # Backwards compatibility.
203  # Configs do not have loggers
204  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
205  self.warpType = 'psfMatched'
206  if self.doSigmaClip and self.statistic != "MEANCLIP":
207  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
208  self.statistic = "MEANCLIP"
209  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
210  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
211  "compute and set a non-zero coadd variance estimate." % (self.statistic))
212 
213  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
214  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
215  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
216  if str(k) not in unstackableStats]
217  raise ValueError("statistic %s is not allowed. Please choose one of %s."
218  % (self.statistic, stackableStats))
219 
220 
221 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
222  """Assemble a coadded image from a set of warps (coadded temporary exposures).
223 
224  We want to assemble a coadded image from a set of Warps (also called
225  coadded temporary exposures or ``coaddTempExps``).
226  Each input Warp covers a patch on the sky and corresponds to a single
227  run/visit/exposure of the covered patch. We provide the task with a list
228  of Warps (``selectDataList``) from which it selects Warps that cover the
229  specified patch (pointed at by ``dataRef``).
230  Each Warp that goes into a coadd will typically have an independent
231  photometric zero-point. Therefore, we must scale each Warp to set it to
232  a common photometric zeropoint. WarpType may be one of 'direct' or
233  'psfMatched', and the boolean configs `config.makeDirect` and
234  `config.makePsfMatched` set which of the warp types will be coadded.
235  The coadd is computed as a mean with optional outlier rejection.
236  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
237  Finally, Warps can have bad 'NaN' pixels which received no input from the
238  source calExps. We interpolate over these bad (NaN) pixels.
239 
240  `AssembleCoaddTask` uses several sub-tasks. These are
241 
242  - `ScaleZeroPointTask`
243  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
244  - `InterpImageTask`
245  - interpolate across bad pixels (NaN) in the final coadd
246 
247  You can retarget these subtasks if you wish.
248 
249  Notes
250  -----
251  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
252  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
253  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
254  no debug variables of its own. Some of the subtasks may support debug
255  variables. See the documentation for the subtasks for further information.
256 
257  Examples
258  --------
259  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
260  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
261  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
262  inputs: a data reference to the tract patch and filter to be coadded, and
263  a list of Warps to attempt to coadd. These are specified using ``--id`` and
264  ``--selectId``, respectively:
265 
266  .. code-block:: none
267 
268  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
269  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
270 
271  Only the Warps that cover the specified tract and patch will be coadded.
272  A list of the available optional arguments can be obtained by calling
273  ``assembleCoadd.py`` with the ``--help`` command line argument:
274 
275  .. code-block:: none
276 
277  assembleCoadd.py --help
278 
279  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
280  multi-band processing, we will generate the HSC-I & -R band coadds from
281  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
282  assuming that the lsst stack has been already set up, we must set up the
283  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
284  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
285  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
286  coadds, we must first
287 
288  - processCcd
289  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
290  - makeSkyMap
291  - create a skymap that covers the area of the sky present in the raw exposures
292  - makeCoaddTempExp
293  - warp the individual calibrated exposures to the tangent plane of the coadd
294 
295  We can perform all of these steps by running
296 
297  .. code-block:: none
298 
299  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
300 
301  This will produce warped exposures for each visit. To coadd the warped
302  data, we call assembleCoadd.py as follows:
303 
304  .. code-block:: none
305 
306  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
307  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
308  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
309  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
310  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
311  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
312  --selectId visit=903988 ccd=24
313 
314  that will process the HSC-I band data. The results are written in
315  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
316 
317  You may also choose to run:
318 
319  .. code-block:: none
320 
321  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
322  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
323  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
324  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
325  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
326  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
327  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
328  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
329 
330  to generate the coadd for the HSC-R band if you are interested in
331  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
332  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
333  rather than `AssembleCoaddTask` to make the coadd.
334  """
335  ConfigClass = AssembleCoaddConfig
336  _DefaultName = "assembleCoadd"
337 
338  def __init__(self, *args, **kwargs):
339  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
340  if args:
341  argNames = ["config", "name", "parentTask", "log"]
342  kwargs.update({k: v for k, v in zip(argNames, args)})
343  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
344  "PipelineTask will not take positional args" % argNames, FutureWarning)
345 
346  super().__init__(**kwargs)
347  self.makeSubtask("interpImage")
348  self.makeSubtask("scaleZeroPoint")
349 
350  if self.config.doMaskBrightObjects:
351  mask = afwImage.Mask()
352  try:
353  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
354  except pexExceptions.LsstCppException:
355  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
356  mask.getMaskPlaneDict().keys())
357  del mask
358 
359  self.warpType = self.config.warpType
360 
361  @classmethod
362  def getOutputDatasetTypes(cls, config):
363  """Return output dataset type descriptors
364 
365  Remove output dataset types not produced by the Task
366  """
367  outputTypeDict = super().getOutputDatasetTypes(config)
368  if not config.doNImage:
369  outputTypeDict.pop("nImage", None)
370  return outputTypeDict
371 
372  @classmethod
373  def getInputDatasetTypes(cls, config):
374  """Return input dataset type descriptors
375 
376  Remove input dataset types not used by the Task
377  """
378  inputTypeDict = super().getInputDatasetTypes(config)
379  if not config.doMaskBrightObjects:
380  inputTypeDict.pop("brightObjectMask", None)
381  else: # TODO: remove this else branch after DM-17300
382  raise RuntimeError("Gen3 Bright object masks unavailable. Set config doMaskBrightObjects=False")
383  return inputTypeDict
384 
385  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
386  """Assemble a coadd from a set of Warps.
387 
388  PipelineTask (Gen3) entry point to Coadd a set of Warps.
389  Analogous to `runDataRef`, it prepares all the data products to be
390  passed to `run`, and processes the results before returning to struct
391  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
392  Therefore, its inputs are accessed subregion by subregion
393  by the `lsst.daf.butler.ShimButler` that quacks like a Gen2
394  `lsst.daf.persistence.Butler`. Updates to this method should
395  correspond to an update in `runDataRef` while both entry points
396  are used.
397 
398  Parameters
399  ----------
400  inputData : `dict`
401  Keys are the names of the configs describing input dataset types.
402  Values are input Python-domain data objects (or lists of objects)
403  retrieved from data butler.
404  inputDataIds : `dict`
405  Keys are the names of the configs describing input dataset types.
406  Values are DataIds (or lists of DataIds) that task consumes for
407  corresponding dataset type.
408  outputDataIds : `dict`
409  Keys are the names of the configs describing input dataset types.
410  Values are DataIds (or lists of DataIds) that task is to produce
411  for corresponding dataset type.
412  butler : `lsst.daf.butler.Butler`
413  Gen3 Butler object for fetching additional data products before
414  running the Task
415 
416  Returns
417  -------
418  result : `lsst.pipe.base.Struct`
419  Result struct with components:
420 
421  - ``coaddExposure`` : coadded exposure (``lsst.afw.image.Exposure``)
422  - ``nImage``: N Image (``lsst.afw.image.Image``)
423  """
424  # Construct skyInfo expected by run
425  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
426  skyMap = inputData["skyMap"]
427  outputDataId = next(iter(outputDataIds.values()))
428  inputData['skyInfo'] = makeSkyInfo(skyMap,
429  tractId=outputDataId['tract'],
430  patchId=outputDataId['patch'])
431 
432  # Construct list of input Shim DataRefs that quack like Gen2 DataRefs
433  butlerShim = ShimButler(butler)
434  warpRefList = [butlerShim.dataRef(self.config.inputWarps.name, dataId=dataId)
435  for dataId in inputDataIds['inputWarps']]
436 
437  # Construct output Shim DataRef
438  patchRef = butlerShim.dataRef(self.config.coaddExposure.name, dataId=outputDataIds['coaddExposure'])
439 
440  # Perform same middle steps as `runDataRef` does
441  inputs = self.prepareInputs(warpRefList)
442  self.log.info("Found %d %s", len(inputs.tempExpRefList),
443  self.getTempExpDatasetName(self.warpType))
444  if len(inputs.tempExpRefList) == 0:
445  self.log.warn("No coadd temporary exposures found")
446  return
447 
448  supplementaryData = self.makeSupplementaryDataGen3(inputData, inputDataIds, outputDataIds, butler)
449 
450  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
451  inputs.weightList, supplementaryData=supplementaryData)
452 
453  self.processResults(retStruct.coaddExposure, patchRef)
454  return retStruct
455 
456  @pipeBase.timeMethod
457  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
458  """Assemble a coadd from a set of Warps.
459 
460  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
461  Compute weights to be applied to each Warp and
462  find scalings to match the photometric zeropoint to a reference Warp.
463  Assemble the Warps using `run`. Interpolate over NaNs and
464  optionally write the coadd to disk. Return the coadded exposure.
465 
466  Parameters
467  ----------
468  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
469  Data reference defining the patch for coaddition and the
470  reference Warp (if ``config.autoReference=False``).
471  Used to access the following data products:
472  - ``self.config.coaddName + "Coadd_skyMap"``
473  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
474  - ``self.config.coaddName + "Coadd"``
475  selectDataList : `list`
476  List of data references to Calexps. Data to be coadded will be
477  selected from this list based on overlap with the patch defined
478  by dataRef, grouped by visit, and converted to a list of data
479  references to warps.
480  warpRefList : `list`
481  List of data references to Warps to be coadded.
482  Note: `warpRefList` is just the new name for `tempExpRefList`.
483 
484  Returns
485  -------
486  retStruct : `lsst.pipe.base.Struct`
487  Result struct with components:
488 
489  - ``coaddExposure``: coadded exposure (``Exposure``).
490  - ``nImage``: exposure count image (``Image``).
491  """
492  if selectDataList and warpRefList:
493  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
494  "and which to use is ambiguous. Please pass only one.")
495 
496  skyInfo = self.getSkyInfo(dataRef)
497  if warpRefList is None:
498  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
499  if len(calExpRefList) == 0:
500  self.log.warn("No exposures to coadd")
501  return
502  self.log.info("Coadding %d exposures", len(calExpRefList))
503 
504  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
505 
506  inputData = self.prepareInputs(warpRefList)
507  self.log.info("Found %d %s", len(inputData.tempExpRefList),
508  self.getTempExpDatasetName(self.warpType))
509  if len(inputData.tempExpRefList) == 0:
510  self.log.warn("No coadd temporary exposures found")
511  return
512 
513  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
514 
515  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
516  inputData.weightList, supplementaryData=supplementaryData)
517 
518  self.processResults(retStruct.coaddExposure, dataRef)
519  if self.config.doWrite:
520  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
521  dataRef.put(retStruct.coaddExposure, self.getCoaddDatasetName(self.warpType))
522  if self.config.doNImage and retStruct.nImage is not None:
523  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
524 
525  return retStruct
526 
527  def processResults(self, coaddExposure, dataRef):
528  """Interpolate over missing data and mask bright stars.
529 
530  Parameters
531  ----------
532  coaddExposure : `lsst.afw.image.Exposure`
533  The coadded exposure to process.
534  dataRef : `lsst.daf.persistence.ButlerDataRef`
535  Butler data reference for supplementary data.
536  """
537  if self.config.doInterp:
538  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
539  # The variance must be positive; work around for DM-3201.
540  varArray = coaddExposure.variance.array
541  with numpy.errstate(invalid="ignore"):
542  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
543 
544  if self.config.doMaskBrightObjects:
545  brightObjectMasks = self.readBrightObjectMasks(dataRef)
546  self.setBrightObjectMasks(coaddExposure, dataRef.dataId, brightObjectMasks)
547 
548  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
549  """Make additional inputs to run() specific to subclasses (Gen2)
550 
551  Duplicates interface of `runDataRef` method
552  Available to be implemented by subclasses only if they need the
553  coadd dataRef for performing preliminary processing before
554  assembling the coadd.
555 
556  Parameters
557  ----------
558  dataRef : `lsst.daf.persistence.ButlerDataRef`
559  Butler data reference for supplementary data.
560  selectDataList : `list`
561  List of data references to Warps.
562  """
563  pass
564 
565  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
566  """Make additional inputs to run() specific to subclasses (Gen3)
567 
568  Duplicates interface of`adaptArgsAndRun` method.
569  Available to be implemented by subclasses only if they need the
570  coadd dataRef for performing preliminary processing before
571  assembling the coadd.
572 
573  Parameters
574  ----------
575  inputData : `dict`
576  Keys are the names of the configs describing input dataset types.
577  Values are input Python-domain data objects (or lists of objects)
578  retrieved from data butler.
579  inputDataIds : `dict`
580  Keys are the names of the configs describing input dataset types.
581  Values are DataIds (or lists of DataIds) that task consumes for
582  corresponding dataset type.
583  DataIds are guaranteed to match data objects in ``inputData``.
584  outputDataIds : `dict`
585  Keys are the names of the configs describing input dataset types.
586  Values are DataIds (or lists of DataIds) that task is to produce
587  for corresponding dataset type.
588  butler : `lsst.daf.butler.Butler`
589  Gen3 Butler object for fetching additional data products before
590  running the Task
591 
592  Returns
593  -------
594  result : `lsst.pipe.base.Struct`
595  Contains whatever additional data the subclass's `run` method needs
596  """
597  pass
598 
599  def getTempExpRefList(self, patchRef, calExpRefList):
600  """Generate list data references corresponding to warped exposures
601  that lie within the patch to be coadded.
602 
603  Parameters
604  ----------
605  patchRef : `dataRef`
606  Data reference for patch.
607  calExpRefList : `list`
608  List of data references for input calexps.
609 
610  Returns
611  -------
612  tempExpRefList : `list`
613  List of Warp/CoaddTempExp data references.
614  """
615  butler = patchRef.getButler()
616  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
617  self.getTempExpDatasetName(self.warpType))
618  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
619  g, groupData.keys) for
620  g in groupData.groups.keys()]
621  return tempExpRefList
622 
623  def prepareInputs(self, refList):
624  """Prepare the input warps for coaddition by measuring the weight for
625  each warp and the scaling for the photometric zero point.
626 
627  Each Warp has its own photometric zeropoint and background variance.
628  Before coadding these Warps together, compute a scale factor to
629  normalize the photometric zeropoint and compute the weight for each Warp.
630 
631  Parameters
632  ----------
633  refList : `list`
634  List of data references to tempExp
635 
636  Returns
637  -------
638  result : `lsst.pipe.base.Struct`
639  Result struct with components:
640 
641  - ``tempExprefList``: `list` of data references to tempExp.
642  - ``weightList``: `list` of weightings.
643  - ``imageScalerList``: `list` of image scalers.
644  """
645  statsCtrl = afwMath.StatisticsControl()
646  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
647  statsCtrl.setNumIter(self.config.clipIter)
648  statsCtrl.setAndMask(self.getBadPixelMask())
649  statsCtrl.setNanSafe(True)
650  # compute tempExpRefList: a list of tempExpRef that actually exist
651  # and weightList: a list of the weight of the associated coadd tempExp
652  # and imageScalerList: a list of scale factors for the associated coadd tempExp
653  tempExpRefList = []
654  weightList = []
655  imageScalerList = []
656  tempExpName = self.getTempExpDatasetName(self.warpType)
657  for tempExpRef in refList:
658  if not tempExpRef.datasetExists(tempExpName):
659  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
660  continue
661 
662  tempExp = tempExpRef.get(tempExpName, immediate=True)
663  maskedImage = tempExp.getMaskedImage()
664  imageScaler = self.scaleZeroPoint.computeImageScaler(
665  exposure=tempExp,
666  dataRef=tempExpRef,
667  )
668  try:
669  imageScaler.scaleMaskedImage(maskedImage)
670  except Exception as e:
671  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
672  continue
673  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
674  afwMath.MEANCLIP, statsCtrl)
675  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
676  weight = 1.0 / float(meanVar)
677  if not numpy.isfinite(weight):
678  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
679  continue
680  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
681 
682  del maskedImage
683  del tempExp
684 
685  tempExpRefList.append(tempExpRef)
686  weightList.append(weight)
687  imageScalerList.append(imageScaler)
688 
689  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
690  imageScalerList=imageScalerList)
691 
692  def prepareStats(self, mask=None):
693  """Prepare the statistics for coadding images.
694 
695  Parameters
696  ----------
697  mask : `int`, optional
698  Bit mask value to exclude from coaddition.
699 
700  Returns
701  -------
702  stats : `lsst.pipe.base.Struct`
703  Statistics structure with the following fields:
704 
705  - ``statsCtrl``: Statistics control object for coadd
706  (`lsst.afw.math.StatisticsControl`)
707  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
708  """
709  if mask is None:
710  mask = self.getBadPixelMask()
711  statsCtrl = afwMath.StatisticsControl()
712  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
713  statsCtrl.setNumIter(self.config.clipIter)
714  statsCtrl.setAndMask(mask)
715  statsCtrl.setNanSafe(True)
716  statsCtrl.setWeighted(True)
717  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
718  for plane, threshold in self.config.maskPropagationThresholds.items():
719  bit = afwImage.Mask.getMaskPlane(plane)
720  statsCtrl.setMaskPropagationThreshold(bit, threshold)
721  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
722  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
723 
724  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
725  altMaskList=None, mask=None, supplementaryData=None):
726  """Assemble a coadd from input warps
727 
728  Assemble the coadd using the provided list of coaddTempExps. Since
729  the full coadd covers a patch (a large area), the assembly is
730  performed over small areas on the image at a time in order to
731  conserve memory usage. Iterate over subregions within the outer
732  bbox of the patch using `assembleSubregion` to stack the corresponding
733  subregions from the coaddTempExps with the statistic specified.
734  Set the edge bits the coadd mask based on the weight map.
735 
736  Parameters
737  ----------
738  skyInfo : `lsst.pipe.base.Struct`
739  Struct with geometric information about the patch.
740  tempExpRefList : `list`
741  List of data references to Warps (previously called CoaddTempExps).
742  imageScalerList : `list`
743  List of image scalers.
744  weightList : `list`
745  List of weights
746  altMaskList : `list`, optional
747  List of alternate masks to use rather than those stored with
748  tempExp.
749  mask : `lsst.afw.image.Mask`, optional
750  Mask to ignore when coadding
751  supplementaryData : lsst.pipe.base.Struct, optional
752  Struct with additional data products needed to assemble coadd.
753  Only used by subclasses that implement `makeSupplementaryData`
754  and override `run`.
755 
756  Returns
757  -------
758  result : `lsst.pipe.base.Struct`
759  Result struct with components:
760 
761  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
762  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
763  """
764  tempExpName = self.getTempExpDatasetName(self.warpType)
765  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
766  stats = self.prepareStats(mask=mask)
767 
768  if altMaskList is None:
769  altMaskList = [None]*len(tempExpRefList)
770 
771  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
772  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
773  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
774  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
775  coaddMaskedImage = coaddExposure.getMaskedImage()
776  subregionSizeArr = self.config.subregionSize
777  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
778  # if nImage is requested, create a zero one which can be passed to assembleSubregion
779  if self.config.doNImage:
780  nImage = afwImage.ImageU(skyInfo.bbox)
781  else:
782  nImage = None
783  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
784  try:
785  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
786  weightList, altMaskList, stats.flags, stats.ctrl,
787  nImage=nImage)
788  except Exception as e:
789  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
790 
791  self.setInexactPsf(coaddMaskedImage.getMask())
792  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
793  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
794  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
795  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
796 
797  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
798  """Set the metadata for the coadd.
799 
800  This basic implementation sets the filter from the first input.
801 
802  Parameters
803  ----------
804  coaddExposure : `lsst.afw.image.Exposure`
805  The target exposure for the coadd.
806  tempExpRefList : `list`
807  List of data references to tempExp.
808  weightList : `list`
809  List of weights.
810  """
811  assert len(tempExpRefList) == len(weightList), "Length mismatch"
812  tempExpName = self.getTempExpDatasetName(self.warpType)
813  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
814  # (and we need more than just the PropertySet that contains the header), which is not possible
815  # with the current butler (see #2777).
816  tempExpList = [tempExpRef.get(tempExpName + "_sub",
817  bbox=afwGeom.Box2I(coaddExposure.getBBox().getMin(),
818  afwGeom.Extent2I(1, 1)), immediate=True)
819  for tempExpRef in tempExpRefList]
820  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
821 
822  coaddExposure.setFilter(tempExpList[0].getFilter())
823  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
824  coaddInputs.ccds.reserve(numCcds)
825  coaddInputs.visits.reserve(len(tempExpList))
826 
827  for tempExp, weight in zip(tempExpList, weightList):
828  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
829 
830  if self.config.doUsePsfMatchedPolygons:
831  self.shrinkValidPolygons(coaddInputs)
832 
833  coaddInputs.visits.sort()
834  if self.warpType == "psfMatched":
835  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
836  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
837  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
838  # having the maximum width (sufficient because square)
839  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
840  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
841  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
842  else:
843  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
844  self.config.coaddPsf.makeControl())
845  coaddExposure.setPsf(psf)
846  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
847  coaddExposure.getWcs())
848  coaddExposure.getInfo().setApCorrMap(apCorrMap)
849  if self.config.doAttachTransmissionCurve:
850  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
851  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
852 
853  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
854  altMaskList, statsFlags, statsCtrl, nImage=None):
855  """Assemble the coadd for a sub-region.
856 
857  For each coaddTempExp, check for (and swap in) an alternative mask
858  if one is passed. Remove mask planes listed in
859  `config.removeMaskPlanes`. Finally, stack the actual exposures using
860  `lsst.afw.math.statisticsStack` with the statistic specified by
861  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
862  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
863  an N-sigma clipped mean where N and iterations are specified by
864  statsCtrl. Assign the stacked subregion back to the coadd.
865 
866  Parameters
867  ----------
868  coaddExposure : `lsst.afw.image.Exposure`
869  The target exposure for the coadd.
870  bbox : `lsst.afw.geom.Box`
871  Sub-region to coadd.
872  tempExpRefList : `list`
873  List of data reference to tempExp.
874  imageScalerList : `list`
875  List of image scalers.
876  weightList : `list`
877  List of weights.
878  altMaskList : `list`
879  List of alternate masks to use rather than those stored with
880  tempExp, or None. Each element is dict with keys = mask plane
881  name to which to add the spans.
882  statsFlags : `lsst.afw.math.Property`
883  Property object for statistic for coadd.
884  statsCtrl : `lsst.afw.math.StatisticsControl`
885  Statistics control object for coadd.
886  nImage : `lsst.afw.image.ImageU`, optional
887  Keeps track of exposure count for each pixel.
888  """
889  self.log.debug("Computing coadd over %s", bbox)
890  tempExpName = self.getTempExpDatasetName(self.warpType)
891  coaddExposure.mask.addMaskPlane("REJECTED")
892  coaddExposure.mask.addMaskPlane("CLIPPED")
893  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
894  maskMap = self.setRejectedMaskMapping(statsCtrl)
895  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
896  maskedImageList = []
897  if nImage is not None:
898  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
899  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
900  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
901  maskedImage = exposure.getMaskedImage()
902  mask = maskedImage.getMask()
903  if altMask is not None:
904  self.applyAltMaskPlanes(mask, altMask)
905  imageScaler.scaleMaskedImage(maskedImage)
906 
907  # Add 1 for each pixel which is not excluded by the exclude mask.
908  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
909  if nImage is not None:
910  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
911  if self.config.removeMaskPlanes:
912  self.removeMaskPlanes(maskedImage)
913  maskedImageList.append(maskedImage)
914 
915  with self.timer("stack"):
916  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
917  clipped, # also set output to CLIPPED if sigma-clipped
918  maskMap)
919  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
920  if nImage is not None:
921  nImage.assign(subNImage, bbox)
922 
923  def removeMaskPlanes(self, maskedImage):
924  """Unset the mask of an image for mask planes specified in the config.
925 
926  Parameters
927  ----------
928  maskedImage : `lsst.afw.image.MaskedImage`
929  The masked image to be modified.
930  """
931  mask = maskedImage.getMask()
932  for maskPlane in self.config.removeMaskPlanes:
933  try:
934  mask &= ~mask.getPlaneBitMask(maskPlane)
935  except Exception as e:
936  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.args[0])
937 
938  @staticmethod
939  def setRejectedMaskMapping(statsCtrl):
940  """Map certain mask planes of the warps to new planes for the coadd.
941 
942  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
943  or CLIPPED, set it to REJECTED on the coadd.
944  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
945  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
946 
947  Parameters
948  ----------
949  statsCtrl : `lsst.afw.math.StatisticsControl`
950  Statistics control object for coadd
951 
952  Returns
953  -------
954  maskMap : `list` of `tuple` of `int`
955  A list of mappings of mask planes of the warped exposures to
956  mask planes of the coadd.
957  """
958  edge = afwImage.Mask.getPlaneBitMask("EDGE")
959  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
960  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
961  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
962  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
963  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
964  (clipped, clipped)]
965  return maskMap
966 
967  def applyAltMaskPlanes(self, mask, altMaskSpans):
968  """Apply in place alt mask formatted as SpanSets to a mask.
969 
970  Parameters
971  ----------
972  mask : `lsst.afw.image.Mask`
973  Original mask.
974  altMaskSpans : `dict`
975  SpanSet lists to apply. Each element contains the new mask
976  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
977  and list of SpanSets to apply to the mask.
978 
979  Returns
980  -------
981  mask : `lsst.afw.image.Mask`
982  Updated mask.
983  """
984  if self.config.doUsePsfMatchedPolygons:
985  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
986  # Clear away any other masks outside the validPolygons. These pixels are no longer
987  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
988  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
989  # This mask-clearing step must only occur *before* applying the new masks below
990  for spanSet in altMaskSpans['NO_DATA']:
991  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
992 
993  for plane, spanSetList in altMaskSpans.items():
994  maskClipValue = mask.addMaskPlane(plane)
995  for spanSet in spanSetList:
996  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
997  return mask
998 
999  def shrinkValidPolygons(self, coaddInputs):
1000  """Shrink coaddInputs' ccds' ValidPolygons in place.
1001 
1002  Either modify each ccd's validPolygon in place, or if CoaddInputs
1003  does not have a validPolygon, create one from its bbox.
1004 
1005  Parameters
1006  ----------
1007  coaddInputs : `lsst.afw.image.coaddInputs`
1008  Original mask.
1009 
1010  """
1011  for ccd in coaddInputs.ccds:
1012  polyOrig = ccd.getValidPolygon()
1013  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1014  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1015  if polyOrig:
1016  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1017  else:
1018  validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
1019  ccd.setValidPolygon(validPolygon)
1020 
1021  def readBrightObjectMasks(self, dataRef):
1022  """Retrieve the bright object masks.
1023 
1024  Returns None on failure.
1025 
1026  Parameters
1027  ----------
1028  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1029  A Butler dataRef.
1030 
1031  Returns
1032  -------
1033  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1034  Bright object mask from the Butler object, or None if it cannot
1035  be retrieved.
1036  """
1037  try:
1038  return dataRef.get("brightObjectMask", immediate=True)
1039  except Exception as e:
1040  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1041  return None
1042 
1043  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
1044  """Set the bright object masks.
1045 
1046  Parameters
1047  ----------
1048  exposure : `lsst.afw.image.Exposure`
1049  Exposure under consideration.
1050  dataId : `lsst.daf.persistence.dataId`
1051  Data identifier dict for patch.
1052  brightObjectMasks : `lsst.afw.table`
1053  Table of bright objects to mask.
1054  """
1055  #
1056  # Check the metadata specifying the tract/patch/filter
1057  #
1058  if brightObjectMasks is None:
1059  self.log.warn("Unable to apply bright object mask: none supplied")
1060  return
1061  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1062  md = brightObjectMasks.table.getMetadata()
1063  for k in dataId:
1064  if not md.exists(k):
1065  self.log.warn("Expected to see %s in metadata", k)
1066  else:
1067  if md.getScalar(k) != dataId[k]:
1068  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
1069 
1070  mask = exposure.getMaskedImage().getMask()
1071  wcs = exposure.getWcs()
1072  plateScale = wcs.getPixelScale().asArcseconds()
1073 
1074  for rec in brightObjectMasks:
1075  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
1076  if rec["type"] == "box":
1077  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1078  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1079  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1080 
1081  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
1082  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
1083 
1084  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1085  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1086  spans = afwGeom.SpanSet(bbox)
1087  elif rec["type"] == "circle":
1088  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1089  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1090  else:
1091  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1092  continue
1093  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1094 
1095  def setInexactPsf(self, mask):
1096  """Set INEXACT_PSF mask plane.
1097 
1098  If any of the input images isn't represented in the coadd (due to
1099  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1100  these pixels.
1101 
1102  Parameters
1103  ----------
1104  mask : `lsst.afw.image.Mask`
1105  Coadded exposure's mask, modified in-place.
1106  """
1107  mask.addMaskPlane("INEXACT_PSF")
1108  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1109  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1110  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1111  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1112  array = mask.getArray()
1113  selected = array & (sensorEdge | clipped | rejected) > 0
1114  array[selected] |= inexactPsf
1115 
1116  @classmethod
1117  def _makeArgumentParser(cls):
1118  """Create an argument parser.
1119  """
1120  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1121  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
1122  cls.ConfigClass().warpType + "Warp",
1123  help="data ID, e.g. --id tract=12345 patch=1,2",
1124  ContainerClass=AssembleCoaddDataIdContainer)
1125  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1126  ContainerClass=SelectDataIdContainer)
1127  return parser
1128 
1129  @staticmethod
1130  def _subBBoxIter(bbox, subregionSize):
1131  """Iterate over subregions of a bbox.
1132 
1133  Parameters
1134  ----------
1135  bbox : `lsst.afw.geom.Box2I`
1136  Bounding box over which to iterate.
1137  subregionSize: `lsst.afw.geom.Extent2I`
1138  Size of sub-bboxes.
1139 
1140  Yields
1141  ------
1142  subBBox : `lsst.afw.geom.Box2I`
1143  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1144  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1145  the edges of ``bbox``, but it will never be empty.
1146  """
1147  if bbox.isEmpty():
1148  raise RuntimeError("bbox %s is empty" % (bbox,))
1149  if subregionSize[0] < 1 or subregionSize[1] < 1:
1150  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1151 
1152  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1153  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1154  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
1155  subBBox.clip(bbox)
1156  if subBBox.isEmpty():
1157  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1158  "colShift=%s, rowShift=%s" %
1159  (bbox, subregionSize, colShift, rowShift))
1160  yield subBBox
1161 
1162 
1163 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1164  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1165  """
1166 
1167  def makeDataRefList(self, namespace):
1168  """Make self.refList from self.idList.
1169 
1170  Parameters
1171  ----------
1172  namespace
1173  Results of parsing command-line (with ``butler`` and ``log`` elements).
1174  """
1175  datasetType = namespace.config.coaddName + "Coadd"
1176  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1177 
1178  for dataId in self.idList:
1179  # tract and patch are required
1180  for key in keysCoadd:
1181  if key not in dataId:
1182  raise RuntimeError("--id must include " + key)
1183 
1184  dataRef = namespace.butler.dataRef(
1185  datasetType=datasetType,
1186  dataId=dataId,
1187  )
1188  self.refList.append(dataRef)
1189 
1190 
1191 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1192  """Function to count the number of pixels with a specific mask in a
1193  footprint.
1194 
1195  Find the intersection of mask & footprint. Count all pixels in the mask
1196  that are in the intersection that have bitmask set but do not have
1197  ignoreMask set. Return the count.
1198 
1199  Parameters
1200  ----------
1201  mask : `lsst.afw.image.Mask`
1202  Mask to define intersection region by.
1203  footprint : `lsst.afw.detection.Footprint`
1204  Footprint to define the intersection region by.
1205  bitmask
1206  Specific mask that we wish to count the number of occurances of.
1207  ignoreMask
1208  Pixels to not consider.
1209 
1210  Returns
1211  -------
1212  result : `int`
1213  Count of number of pixels in footprint with specified mask.
1214  """
1215  bbox = footprint.getBBox()
1216  bbox.clip(mask.getBBox(afwImage.PARENT))
1217  fp = afwImage.Mask(bbox)
1218  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1219  footprint.spans.setMask(fp, bitmask)
1220  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1221  (subMask.getArray() & ignoreMask) == 0).sum()
1222 
1223 
1225  """Configuration parameters for the SafeClipAssembleCoaddTask.
1226  """
1227  clipDetection = pexConfig.ConfigurableField(
1228  target=SourceDetectionTask,
1229  doc="Detect sources on difference between unclipped and clipped coadd")
1230  minClipFootOverlap = pexConfig.Field(
1231  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1232  dtype=float,
1233  default=0.6
1234  )
1235  minClipFootOverlapSingle = pexConfig.Field(
1236  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1237  "clipped when only one visit overlaps",
1238  dtype=float,
1239  default=0.5
1240  )
1241  minClipFootOverlapDouble = pexConfig.Field(
1242  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1243  "clipped when two visits overlap",
1244  dtype=float,
1245  default=0.45
1246  )
1247  maxClipFootOverlapDouble = pexConfig.Field(
1248  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1249  "considering two visits",
1250  dtype=float,
1251  default=0.15
1252  )
1253  minBigOverlap = pexConfig.Field(
1254  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1255  "when labeling clipped footprints",
1256  dtype=int,
1257  default=100
1258  )
1259 
1260  def setDefaults(self):
1261  """Set default values for clipDetection.
1262 
1263  Notes
1264  -----
1265  The numeric values for these configuration parameters were
1266  empirically determined, future work may further refine them.
1267  """
1268  AssembleCoaddConfig.setDefaults(self)
1269  self.clipDetection.doTempLocalBackground = False
1270  self.clipDetection.reEstimateBackground = False
1271  self.clipDetection.returnOriginalFootprints = False
1272  self.clipDetection.thresholdPolarity = "both"
1273  self.clipDetection.thresholdValue = 2
1274  self.clipDetection.nSigmaToGrow = 2
1275  self.clipDetection.minPixels = 4
1276  self.clipDetection.isotropicGrow = True
1277  self.clipDetection.thresholdType = "pixel_stdev"
1278  self.sigmaClip = 1.5
1279  self.clipIter = 3
1280  self.statistic = "MEAN"
1281 
1282  def validate(self):
1283  if self.doSigmaClip:
1284  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1285  "Ignoring doSigmaClip.")
1286  self.doSigmaClip = False
1287  if self.statistic != "MEAN":
1288  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1289  "(%s chosen). Please set statistic to MEAN."
1290  % (self.statistic))
1291  AssembleCoaddTask.ConfigClass.validate(self)
1292 
1293 
1295  """Assemble a coadded image from a set of coadded temporary exposures,
1296  being careful to clip & flag areas with potential artifacts.
1297 
1298  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1299  we clip outliers). The problem with doing this is that when computing the
1300  coadd PSF at a given location, individual visit PSFs from visits with
1301  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1302  In this task, we correct for this behavior by creating a new
1303  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1304  coaddTempExps and the final coadd where
1305 
1306  i. difference imaging suggests that there is an outlier and
1307  ii. this outlier appears on only one or two images.
1308 
1309  Such regions will not contribute to the final coadd. Furthermore, any
1310  routine to determine the coadd PSF can now be cognizant of clipped regions.
1311  Note that the algorithm implemented by this task is preliminary and works
1312  correctly for HSC data. Parameter modifications and or considerable
1313  redesigning of the algorithm is likley required for other surveys.
1314 
1315  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1316  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1317  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1318  if you wish.
1319 
1320  Notes
1321  -----
1322  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1323  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1324  see `baseDebug` for more about ``debug.py`` files.
1325  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1326  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1327  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1328  for further information.
1329 
1330  Examples
1331  --------
1332  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1333  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1334  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1335 
1336  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1337  and filter to be coadded (specified using
1338  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1339  along with a list of coaddTempExps to attempt to coadd (specified using
1340  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1341  Only the coaddTempExps that cover the specified tract and patch will be
1342  coadded. A list of the available optional arguments can be obtained by
1343  calling assembleCoadd.py with the --help command line argument:
1344 
1345  .. code-block:: none
1346 
1347  assembleCoadd.py --help
1348 
1349  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1350  context of multi-band processing, we will generate the HSC-I & -R band
1351  coadds from HSC engineering test data provided in the ci_hsc package.
1352  To begin, assuming that the lsst stack has been already set up, we must
1353  set up the obs_subaru and ci_hsc packages. This defines the environment
1354  variable $CI_HSC_DIR and points at the location of the package. The raw
1355  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1356  the coadds, we must first
1357 
1358  - ``processCcd``
1359  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1360  - ``makeSkyMap``
1361  create a skymap that covers the area of the sky present in the raw exposures
1362  - ``makeCoaddTempExp``
1363  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1364 
1365  We can perform all of these steps by running
1366 
1367  .. code-block:: none
1368 
1369  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1370 
1371  This will produce warped coaddTempExps for each visit. To coadd the
1372  warped data, we call ``assembleCoadd.py`` as follows:
1373 
1374  .. code-block:: none
1375 
1376  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1377  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1378  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1379  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1380  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1381  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1382  --selectId visit=903988 ccd=24
1383 
1384  This will process the HSC-I band data. The results are written in
1385  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1386 
1387  You may also choose to run:
1388 
1389  .. code-block:: none
1390 
1391  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1392  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1393  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1394  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1395  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1396  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1397  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1398  --selectId visit=903346 ccd=12
1399 
1400  to generate the coadd for the HSC-R band if you are interested in following
1401  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1402  """
1403  ConfigClass = SafeClipAssembleCoaddConfig
1404  _DefaultName = "safeClipAssembleCoadd"
1405 
1406  def __init__(self, *args, **kwargs):
1407  AssembleCoaddTask.__init__(self, *args, **kwargs)
1408  schema = afwTable.SourceTable.makeMinimalSchema()
1409  self.makeSubtask("clipDetection", schema=schema)
1410 
1411  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1412  """Assemble the coadd for a region.
1413 
1414  Compute the difference of coadds created with and without outlier
1415  rejection to identify coadd pixels that have outlier values in some
1416  individual visits.
1417  Detect clipped regions on the difference image and mark these regions
1418  on the one or two individual coaddTempExps where they occur if there
1419  is significant overlap between the clipped region and a source. This
1420  leaves us with a set of footprints from the difference image that have
1421  been identified as having occured on just one or two individual visits.
1422  However, these footprints were generated from a difference image. It
1423  is conceivable for a large diffuse source to have become broken up
1424  into multiple footprints acrosss the coadd difference in this process.
1425  Determine the clipped region from all overlapping footprints from the
1426  detected sources in each visit - these are big footprints.
1427  Combine the small and big clipped footprints and mark them on a new
1428  bad mask plane.
1429  Generate the coadd using `AssembleCoaddTask.run` without outlier
1430  removal. Clipped footprints will no longer make it into the coadd
1431  because they are marked in the new bad mask plane.
1432 
1433  Parameters
1434  ----------
1435  skyInfo : `lsst.pipe.base.Struct`
1436  Patch geometry information, from getSkyInfo
1437  tempExpRefList : `list`
1438  List of data reference to tempExp
1439  imageScalerList : `list`
1440  List of image scalers
1441  weightList : `list`
1442  List of weights
1443 
1444  Returns
1445  -------
1446  result : `lsst.pipe.base.Struct`
1447  Result struct with components:
1448 
1449  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1450  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
1451 
1452  Notes
1453  -----
1454  args and kwargs are passed but ignored in order to match the call
1455  signature expected by the parent task.
1456  """
1457  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1458  mask = exp.getMaskedImage().getMask()
1459  mask.addMaskPlane("CLIPPED")
1460 
1461  result = self.detectClip(exp, tempExpRefList)
1462 
1463  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1464 
1465  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1466  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1467  # Append big footprints from individual Warps to result.clipSpans
1468  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1469  result.detectionFootprints, maskClipValue, maskDetValue,
1470  exp.getBBox())
1471  # Create mask of the current clipped footprints
1472  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1473  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1474 
1475  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1476  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1477  maskClip |= maskClipBig
1478 
1479  # Assemble coadd from base class, but ignoring CLIPPED pixels
1480  badMaskPlanes = self.config.badMaskPlanes[:]
1481  badMaskPlanes.append("CLIPPED")
1482  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1483  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1484  result.clipSpans, mask=badPixelMask)
1485 
1486  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1487  """Return an exposure that contains the difference between unclipped
1488  and clipped coadds.
1489 
1490  Generate a difference image between clipped and unclipped coadds.
1491  Compute the difference image by subtracting an outlier-clipped coadd
1492  from an outlier-unclipped coadd. Return the difference image.
1493 
1494  Parameters
1495  ----------
1496  skyInfo : `lsst.pipe.base.Struct`
1497  Patch geometry information, from getSkyInfo
1498  tempExpRefList : `list`
1499  List of data reference to tempExp
1500  imageScalerList : `list`
1501  List of image scalers
1502  weightList : `list`
1503  List of weights
1504 
1505  Returns
1506  -------
1507  exp : `lsst.afw.image.Exposure`
1508  Difference image of unclipped and clipped coadd wrapped in an Exposure
1509  """
1510  # Clone and upcast self.config because current self.config is frozen
1511  config = AssembleCoaddConfig()
1512  # getattr necessary because subtasks do not survive Config.toDict()
1513  configIntersection = {k: getattr(self.config, k)
1514  for k, v in self.config.toDict().items() if (k in config.keys())}
1515  config.update(**configIntersection)
1516 
1517  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1518  config.statistic = 'MEAN'
1519  task = AssembleCoaddTask(config=config)
1520  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1521 
1522  config.statistic = 'MEANCLIP'
1523  task = AssembleCoaddTask(config=config)
1524  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1525 
1526  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1527  coaddDiff -= coaddClip.getMaskedImage()
1528  exp = afwImage.ExposureF(coaddDiff)
1529  exp.setPsf(coaddMean.getPsf())
1530  return exp
1531 
1532  def detectClip(self, exp, tempExpRefList):
1533  """Detect clipped regions on an exposure and set the mask on the
1534  individual tempExp masks.
1535 
1536  Detect footprints in the difference image after smoothing the
1537  difference image with a Gaussian kernal. Identify footprints that
1538  overlap with one or two input ``coaddTempExps`` by comparing the
1539  computed overlap fraction to thresholds set in the config. A different
1540  threshold is applied depending on the number of overlapping visits
1541  (restricted to one or two). If the overlap exceeds the thresholds,
1542  the footprint is considered "CLIPPED" and is marked as such on the
1543  coaddTempExp. Return a struct with the clipped footprints, the indices
1544  of the ``coaddTempExps`` that end up overlapping with the clipped
1545  footprints, and a list of new masks for the ``coaddTempExps``.
1546 
1547  Parameters
1548  ----------
1549  exp : `lsst.afw.image.Exposure`
1550  Exposure to run detection on.
1551  tempExpRefList : `list`
1552  List of data reference to tempExp.
1553 
1554  Returns
1555  -------
1556  result : `lsst.pipe.base.Struct`
1557  Result struct with components:
1558 
1559  - ``clipFootprints``: list of clipped footprints.
1560  - ``clipIndices``: indices for each ``clippedFootprint`` in
1561  ``tempExpRefList``.
1562  - ``clipSpans``: List of dictionaries containing spanSet lists
1563  to clip. Each element contains the new maskplane name
1564  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1565  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1566  compressed into footprints.
1567  """
1568  mask = exp.getMaskedImage().getMask()
1569  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1570  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1571  # Merge positive and negative together footprints together
1572  fpSet.positive.merge(fpSet.negative)
1573  footprints = fpSet.positive
1574  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1575  ignoreMask = self.getBadPixelMask()
1576 
1577  clipFootprints = []
1578  clipIndices = []
1579  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1580 
1581  # for use by detectClipBig
1582  visitDetectionFootprints = []
1583 
1584  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1585  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1586  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1587 
1588  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1589  for i, warpRef in enumerate(tempExpRefList):
1590  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1591  immediate=True).getMaskedImage().getMask()
1592  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1593  afwImage.PARENT, True)
1594  maskVisitDet &= maskDetValue
1595  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1596  visitDetectionFootprints.append(visitFootprints)
1597 
1598  for j, footprint in enumerate(footprints.getFootprints()):
1599  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1600  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1601 
1602  # build a list of clipped spans for each visit
1603  for j, footprint in enumerate(footprints.getFootprints()):
1604  nPixel = footprint.getArea()
1605  overlap = [] # hold the overlap with each visit
1606  indexList = [] # index of visit in global list
1607  for i in range(len(tempExpRefList)):
1608  ignore = ignoreArr[i, j]
1609  overlapDet = overlapDetArr[i, j]
1610  totPixel = nPixel - ignore
1611 
1612  # If we have more bad pixels than detection skip
1613  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1614  continue
1615  overlap.append(overlapDet/float(totPixel))
1616  indexList.append(i)
1617 
1618  overlap = numpy.array(overlap)
1619  if not len(overlap):
1620  continue
1621 
1622  keep = False # Should this footprint be marked as clipped?
1623  keepIndex = [] # Which tempExps does the clipped footprint belong to
1624 
1625  # If footprint only has one overlap use a lower threshold
1626  if len(overlap) == 1:
1627  if overlap[0] > self.config.minClipFootOverlapSingle:
1628  keep = True
1629  keepIndex = [0]
1630  else:
1631  # This is the general case where only visit should be clipped
1632  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1633  if len(clipIndex) == 1:
1634  keep = True
1635  keepIndex = [clipIndex[0]]
1636 
1637  # Test if there are clipped objects that overlap two different visits
1638  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1639  if len(clipIndex) == 2 and len(overlap) > 3:
1640  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1641  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1642  keep = True
1643  keepIndex = clipIndex
1644 
1645  if not keep:
1646  continue
1647 
1648  for index in keepIndex:
1649  globalIndex = indexList[index]
1650  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1651 
1652  clipIndices.append(numpy.array(indexList)[keepIndex])
1653  clipFootprints.append(footprint)
1654 
1655  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1656  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1657 
1658  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1659  maskClipValue, maskDetValue, coaddBBox):
1660  """Return individual warp footprints for large artifacts and append
1661  them to ``clipList`` in place.
1662 
1663  Identify big footprints composed of many sources in the coadd
1664  difference that may have originated in a large diffuse source in the
1665  coadd. We do this by indentifying all clipped footprints that overlap
1666  significantly with each source in all the coaddTempExps.
1667 
1668  Parameters
1669  ----------
1670  clipList : `list`
1671  List of alt mask SpanSets with clipping information. Modified.
1672  clipFootprints : `list`
1673  List of clipped footprints.
1674  clipIndices : `list`
1675  List of which entries in tempExpClipList each footprint belongs to.
1676  maskClipValue
1677  Mask value of clipped pixels.
1678  maskDetValue
1679  Mask value of detected pixels.
1680  coaddBBox : `lsst.afw.geom.Box`
1681  BBox of the coadd and warps.
1682 
1683  Returns
1684  -------
1685  bigFootprintsCoadd : `list`
1686  List of big footprints
1687  """
1688  bigFootprintsCoadd = []
1689  ignoreMask = self.getBadPixelMask()
1690  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1691  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1692  for footprint in visitFootprints.getFootprints():
1693  footprint.spans.setMask(maskVisitDet, maskDetValue)
1694 
1695  # build a mask of clipped footprints that are in this visit
1696  clippedFootprintsVisit = []
1697  for foot, clipIndex in zip(clipFootprints, clipIndices):
1698  if index not in clipIndex:
1699  continue
1700  clippedFootprintsVisit.append(foot)
1701  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1702  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1703 
1704  bigFootprintsVisit = []
1705  for foot in visitFootprints.getFootprints():
1706  if foot.getArea() < self.config.minBigOverlap:
1707  continue
1708  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1709  if nCount > self.config.minBigOverlap:
1710  bigFootprintsVisit.append(foot)
1711  bigFootprintsCoadd.append(foot)
1712 
1713  for footprint in bigFootprintsVisit:
1714  clippedSpans["CLIPPED"].append(footprint.spans)
1715 
1716  return bigFootprintsCoadd
1717 
1718 
1720  assembleStaticSkyModel = pexConfig.ConfigurableField(
1721  target=AssembleCoaddTask,
1722  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1723  " naive/first-iteration model of the static sky.",
1724  )
1725  detect = pexConfig.ConfigurableField(
1726  target=SourceDetectionTask,
1727  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1728  )
1729  detectTemplate = pexConfig.ConfigurableField(
1730  target=SourceDetectionTask,
1731  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1732  )
1733  maxNumEpochs = pexConfig.Field(
1734  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1735  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1736  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1737  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1738  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1739  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1740  "than transient and not masked.",
1741  dtype=int,
1742  default=2
1743  )
1744  maxFractionEpochsLow = pexConfig.RangeField(
1745  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1746  "Effective maxNumEpochs = "
1747  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1748  dtype=float,
1749  default=0.4,
1750  min=0., max=1.,
1751  )
1752  maxFractionEpochsHigh = pexConfig.RangeField(
1753  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1754  "Effective maxNumEpochs = "
1755  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1756  dtype=float,
1757  default=0.03,
1758  min=0., max=1.,
1759  )
1760  spatialThreshold = pexConfig.RangeField(
1761  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1762  "temporal criteria. If 0, clip all. If 1, clip none.",
1763  dtype=float,
1764  default=0.5,
1765  min=0., max=1.,
1766  inclusiveMin=True, inclusiveMax=True
1767  )
1768  doScaleWarpVariance = pexConfig.Field(
1769  doc="Rescale Warp variance plane using empirical noise?",
1770  dtype=bool,
1771  default=True,
1772  )
1773  scaleWarpVariance = pexConfig.ConfigurableField(
1774  target=ScaleVarianceTask,
1775  doc="Rescale variance on warps",
1776  )
1777  doPreserveContainedBySource = pexConfig.Field(
1778  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1779  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1780  dtype=bool,
1781  default=True,
1782  )
1783  doPrefilterArtifacts = pexConfig.Field(
1784  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1785  "because they will be excluded anyway. This prevents them from contributing "
1786  "to the outlier epoch count image and potentially being labeled as persistant."
1787  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1788  dtype=bool,
1789  default=True
1790  )
1791  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1792  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1793  dtype=str,
1794  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1795  )
1796  prefilterArtifactsRatio = pexConfig.Field(
1797  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1798  dtype=float,
1799  default=0.05
1800  )
1801  psfMatchedWarps = pipeBase.InputDatasetField(
1802  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1803  "Only PSF-Matched Warps make sense for image subtraction. "
1804  "Therefore, they must be in the InputDatasetField and made available to the task."),
1805  nameTemplate="{inputCoaddName}Coadd_psfMatchedWarp",
1806  storageClass="ExposureF",
1807  dimensions=("Tract", "Patch", "SkyMap", "Visit"),
1808  manualLoad=True
1809  )
1810 
1811  def setDefaults(self):
1812  AssembleCoaddConfig.setDefaults(self)
1813  self.statistic = 'MEAN'
1815 
1816  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1817  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1818  if "EDGE" in self.badMaskPlanes:
1819  self.badMaskPlanes.remove('EDGE')
1820  self.removeMaskPlanes.append('EDGE')
1821  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1822  self.assembleStaticSkyModel.warpType = 'psfMatched'
1823  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1824  self.assembleStaticSkyModel.sigmaClip = 2.5
1825  self.assembleStaticSkyModel.clipIter = 3
1826  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1827  self.assembleStaticSkyModel.doWrite = False
1828  self.detect.doTempLocalBackground = False
1829  self.detect.reEstimateBackground = False
1830  self.detect.returnOriginalFootprints = False
1831  self.detect.thresholdPolarity = "both"
1832  self.detect.thresholdValue = 5
1833  self.detect.nSigmaToGrow = 2
1834  self.detect.minPixels = 4
1835  self.detect.isotropicGrow = True
1836  self.detect.thresholdType = "pixel_stdev"
1837  self.detectTemplate.nSigmaToGrow = 2.0
1838  self.detectTemplate.doTempLocalBackground = False
1839  self.detectTemplate.reEstimateBackground = False
1840  self.detectTemplate.returnOriginalFootprints = False
1841 
1842 
1844  """Assemble a compareWarp coadded image from a set of warps
1845  by masking artifacts detected by comparing PSF-matched warps.
1846 
1847  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1848  we clip outliers). The problem with doing this is that when computing the
1849  coadd PSF at a given location, individual visit PSFs from visits with
1850  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1851  In this task, we correct for this behavior by creating a new badMaskPlane
1852  'CLIPPED' which marks pixels in the individual warps suspected to contain
1853  an artifact. We populate this plane on the input warps by comparing
1854  PSF-matched warps with a PSF-matched median coadd which serves as a
1855  model of the static sky. Any group of pixels that deviates from the
1856  PSF-matched template coadd by more than config.detect.threshold sigma,
1857  is an artifact candidate. The candidates are then filtered to remove
1858  variable sources and sources that are difficult to subtract such as
1859  bright stars. This filter is configured using the config parameters
1860  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1861  the maximum fraction of epochs that the deviation can appear in and still
1862  be considered an artifact. The spatialThreshold is the maximum fraction of
1863  pixels in the footprint of the deviation that appear in other epochs
1864  (where other epochs is defined by the temporalThreshold). If the deviant
1865  region meets this criteria of having a significant percentage of pixels
1866  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1867  set in the mask. These regions will not contribute to the final coadd.
1868  Furthermore, any routine to determine the coadd PSF can now be cognizant
1869  of clipped regions. Note that the algorithm implemented by this task is
1870  preliminary and works correctly for HSC data. Parameter modifications and
1871  or considerable redesigning of the algorithm is likley required for other
1872  surveys.
1873 
1874  ``CompareWarpAssembleCoaddTask`` sub-classes
1875  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1876  as a subtask to generate the TemplateCoadd (the model of the static sky).
1877 
1878  Notes
1879  -----
1880  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1881  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1882  ``baseDebug`` for more about ``debug.py`` files.
1883 
1884  This task supports the following debug variables:
1885 
1886  - ``saveCountIm``
1887  If True then save the Epoch Count Image as a fits file in the `figPath`
1888  - ``figPath``
1889  Path to save the debug fits images and figures
1890 
1891  For example, put something like:
1892 
1893  .. code-block:: python
1894 
1895  import lsstDebug
1896  def DebugInfo(name):
1897  di = lsstDebug.getInfo(name)
1898  if name == "lsst.pipe.tasks.assembleCoadd":
1899  di.saveCountIm = True
1900  di.figPath = "/desired/path/to/debugging/output/images"
1901  return di
1902  lsstDebug.Info = DebugInfo
1903 
1904  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1905  ``--debug`` flag. Some subtasks may have their own debug variables;
1906  see individual Task documentation.
1907 
1908  Examples
1909  --------
1910  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1911  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1912  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1913  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1914  and filter to be coadded (specified using
1915  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1916  along with a list of coaddTempExps to attempt to coadd (specified using
1917  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1918  Only the warps that cover the specified tract and patch will be coadded.
1919  A list of the available optional arguments can be obtained by calling
1920  ``assembleCoadd.py`` with the ``--help`` command line argument:
1921 
1922  .. code-block:: none
1923 
1924  assembleCoadd.py --help
1925 
1926  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1927  context of multi-band processing, we will generate the HSC-I & -R band
1928  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1929  To begin, assuming that the lsst stack has been already set up, we must
1930  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1931  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1932  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1933  directory. To begin assembling the coadds, we must first
1934 
1935  - processCcd
1936  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1937  - makeSkyMap
1938  create a skymap that covers the area of the sky present in the raw exposures
1939  - makeCoaddTempExp
1940  warp the individual calibrated exposures to the tangent plane of the coadd
1941 
1942  We can perform all of these steps by running
1943 
1944  .. code-block:: none
1945 
1946  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1947 
1948  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1949  warped data, we call ``assembleCoadd.py`` as follows:
1950 
1951  .. code-block:: none
1952 
1953  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1954  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1955  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1956  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1957  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1958  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1959  --selectId visit=903988 ccd=24
1960 
1961  This will process the HSC-I band data. The results are written in
1962  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1963  """
1964  ConfigClass = CompareWarpAssembleCoaddConfig
1965  _DefaultName = "compareWarpAssembleCoadd"
1966 
1967  def __init__(self, *args, **kwargs):
1968  AssembleCoaddTask.__init__(self, *args, **kwargs)
1969  self.makeSubtask("assembleStaticSkyModel")
1970  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1971  self.makeSubtask("detect", schema=detectionSchema)
1972  if self.config.doPreserveContainedBySource:
1973  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1974  if self.config.doScaleWarpVariance:
1975  self.makeSubtask("scaleWarpVariance")
1976 
1977  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
1978  """Make inputs specific to Subclass with Gen 3 API
1979 
1980  Calls Gen3 `adaptArgsAndRun` instead of the Gen2 specific `runDataRef`
1981 
1982  Duplicates interface of`adaptArgsAndRun` method.
1983  Available to be implemented by subclasses only if they need the
1984  coadd dataRef for performing preliminary processing before
1985  assembling the coadd.
1986 
1987  Parameters
1988  ----------
1989  inputData : `dict`
1990  Keys are the names of the configs describing input dataset types.
1991  Values are input Python-domain data objects (or lists of objects)
1992  retrieved from data butler.
1993  inputDataIds : `dict`
1994  Keys are the names of the configs describing input dataset types.
1995  Values are DataIds (or lists of DataIds) that task consumes for
1996  corresponding dataset type.
1997  DataIds are guaranteed to match data objects in ``inputData``.
1998  outputDataIds : `dict`
1999  Keys are the names of the configs describing input dataset types.
2000  Values are DataIds (or lists of DataIds) that task is to produce
2001  for corresponding dataset type.
2002  butler : `lsst.daf.butler.Butler`
2003  Gen3 Butler object for fetching additional data products before
2004  running the Task
2005 
2006  Returns
2007  -------
2008  result : `lsst.pipe.base.Struct`
2009  Result struct with components:
2010 
2011  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2012  - ``nImage``: N Image (``lsst.afw.image.Image``)
2013 
2014 
2015  """
2016  templateCoadd = self.assembleStaticSkyModel.adaptArgsAndRun(inputData, inputDataIds,
2017  outputDataIds, butler)
2018  if templateCoadd is None:
2019  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2020 
2021  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2022  nImage=templateCoadd.nImage)
2023 
2024  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2025  """Make inputs specific to Subclass.
2026 
2027  Generate a templateCoadd to use as a native model of static sky to
2028  subtract from warps.
2029 
2030  Parameters
2031  ----------
2032  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2033  Butler dataRef for supplementary data.
2034  selectDataList : `list` (optional)
2035  Optional List of data references to Calexps.
2036  warpRefList : `list` (optional)
2037  Optional List of data references to Warps.
2038 
2039  Returns
2040  -------
2041  result : `lsst.pipe.base.Struct`
2042  Result struct with components:
2043 
2044  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2045  - ``nImage``: N Image (``lsst.afw.image.Image``)
2046  """
2047  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2048  if templateCoadd is None:
2049  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2050 
2051  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2052  nImage=templateCoadd.nImage)
2053 
2054  def _noTemplateMessage(self, warpType):
2055  warpName = (warpType[0].upper() + warpType[1:])
2056  message = """No %(warpName)s warps were found to build the template coadd which is
2057  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2058  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2059  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2060 
2061  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2062  another algorithm like:
2063 
2064  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2065  config.assemble.retarget(SafeClipAssembleCoaddTask)
2066  """ % {"warpName": warpName}
2067  return message
2068 
2069  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2070  supplementaryData, *args, **kwargs):
2071  """Assemble the coadd.
2072 
2073  Find artifacts and apply them to the warps' masks creating a list of
2074  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2075  plane. Then pass these alternative masks to the base class's `run`
2076  method.
2077 
2078  Parameters
2079  ----------
2080  skyInfo : `lsst.pipe.base.Struct`
2081  Patch geometry information.
2082  tempExpRefList : `list`
2083  List of data references to warps.
2084  imageScalerList : `list`
2085  List of image scalers.
2086  weightList : `list`
2087  List of weights.
2088  supplementaryData : `lsst.pipe.base.Struct`
2089  This Struct must contain a ``templateCoadd`` that serves as the
2090  model of the static sky.
2091 
2092  Returns
2093  -------
2094  result : `lsst.pipe.base.Struct`
2095  Result struct with components:
2096 
2097  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
2098  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
2099  """
2100  templateCoadd = supplementaryData.templateCoadd
2101  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
2102  badMaskPlanes = self.config.badMaskPlanes[:]
2103  badMaskPlanes.append("CLIPPED")
2104  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2105 
2106  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2107  spanSetMaskList, mask=badPixelMask)
2108 
2109  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2110  # Psf-Matching moves the real edge inwards
2111  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2112  return result
2113 
2114  def applyAltEdgeMask(self, mask, altMaskList):
2115  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2116 
2117  Parameters
2118  ----------
2119  mask : `lsst.afw.image.Mask`
2120  Original mask.
2121  altMaskList : `list`
2122  List of Dicts containing ``spanSet`` lists.
2123  Each element contains the new mask plane name (e.g. "CLIPPED
2124  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2125  the mask.
2126  """
2127  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2128  for visitMask in altMaskList:
2129  if "EDGE" in visitMask:
2130  for spanSet in visitMask['EDGE']:
2131  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2132 
2133  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2134  """Find artifacts.
2135 
2136  Loop through warps twice. The first loop builds a map with the count
2137  of how many epochs each pixel deviates from the templateCoadd by more
2138  than ``config.chiThreshold`` sigma. The second loop takes each
2139  difference image and filters the artifacts detected in each using
2140  count map to filter out variable sources and sources that are
2141  difficult to subtract cleanly.
2142 
2143  Parameters
2144  ----------
2145  templateCoadd : `lsst.afw.image.Exposure`
2146  Exposure to serve as model of static sky.
2147  tempExpRefList : `list`
2148  List of data references to warps.
2149  imageScalerList : `list`
2150  List of image scalers.
2151 
2152  Returns
2153  -------
2154  altMasks : `list`
2155  List of dicts containing information about CLIPPED
2156  (i.e., artifacts), NO_DATA, and EDGE pixels.
2157  """
2158 
2159  self.log.debug("Generating Count Image, and mask lists.")
2160  coaddBBox = templateCoadd.getBBox()
2161  slateIm = afwImage.ImageU(coaddBBox)
2162  epochCountImage = afwImage.ImageU(coaddBBox)
2163  nImage = afwImage.ImageU(coaddBBox)
2164  spanSetArtifactList = []
2165  spanSetNoDataMaskList = []
2166  spanSetEdgeList = []
2167  badPixelMask = self.getBadPixelMask()
2168 
2169  # mask of the warp diffs should = that of only the warp
2170  templateCoadd.mask.clearAllMaskPlanes()
2171 
2172  if self.config.doPreserveContainedBySource:
2173  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2174  else:
2175  templateFootprints = None
2176 
2177  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2178  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2179  if warpDiffExp is not None:
2180  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2181  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2182  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2183  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2184  fpSet.positive.merge(fpSet.negative)
2185  footprints = fpSet.positive
2186  slateIm.set(0)
2187  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2188 
2189  # Remove artifacts due to defects before they contribute to the epochCountImage
2190  if self.config.doPrefilterArtifacts:
2191  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2192  for spans in spanSetList:
2193  spans.setImage(slateIm, 1, doClip=True)
2194  epochCountImage += slateIm
2195 
2196  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2197  # undergo a second convolution. Pixels with data in the direct warp
2198  # but not in the PSF-matched warp will not have their artifacts detected.
2199  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2200  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2201  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2202  nansMask.setXY0(warpDiffExp.getXY0())
2203  edgeMask = warpDiffExp.mask
2204  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2205  edgeMask.getPlaneBitMask("EDGE")).split()
2206  else:
2207  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2208  # In this case, mask the whole epoch
2209  nansMask = afwImage.MaskX(coaddBBox, 1)
2210  spanSetList = []
2211  spanSetEdgeMask = []
2212 
2213  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2214 
2215  spanSetNoDataMaskList.append(spanSetNoDataMask)
2216  spanSetArtifactList.append(spanSetList)
2217  spanSetEdgeList.append(spanSetEdgeMask)
2218 
2219  if lsstDebug.Info(__name__).saveCountIm:
2220  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2221  epochCountImage.writeFits(path)
2222 
2223  for i, spanSetList in enumerate(spanSetArtifactList):
2224  if spanSetList:
2225  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2226  templateFootprints)
2227  spanSetArtifactList[i] = filteredSpanSetList
2228 
2229  altMasks = []
2230  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2231  altMasks.append({'CLIPPED': artifacts,
2232  'NO_DATA': noData,
2233  'EDGE': edge})
2234  return altMasks
2235 
2236  def prefilterArtifacts(self, spanSetList, exp):
2237  """Remove artifact candidates covered by bad mask plane.
2238 
2239  Any future editing of the candidate list that does not depend on
2240  temporal information should go in this method.
2241 
2242  Parameters
2243  ----------
2244  spanSetList : `list`
2245  List of SpanSets representing artifact candidates.
2246  exp : `lsst.afw.image.Exposure`
2247  Exposure containing mask planes used to prefilter.
2248 
2249  Returns
2250  -------
2251  returnSpanSetList : `list`
2252  List of SpanSets with artifacts.
2253  """
2254  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2255  goodArr = (exp.mask.array & badPixelMask) == 0
2256  returnSpanSetList = []
2257  bbox = exp.getBBox()
2258  x0, y0 = exp.getXY0()
2259  for i, span in enumerate(spanSetList):
2260  y, x = span.clippedTo(bbox).indices()
2261  yIndexLocal = numpy.array(y) - y0
2262  xIndexLocal = numpy.array(x) - x0
2263  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2264  if goodRatio > self.config.prefilterArtifactsRatio:
2265  returnSpanSetList.append(span)
2266  return returnSpanSetList
2267 
2268  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2269  """Filter artifact candidates.
2270 
2271  Parameters
2272  ----------
2273  spanSetList : `list`
2274  List of SpanSets representing artifact candidates.
2275  epochCountImage : `lsst.afw.image.Image`
2276  Image of accumulated number of warpDiff detections.
2277  nImage : `lsst.afw.image.Image`
2278  Image of the accumulated number of total epochs contributing.
2279 
2280  Returns
2281  -------
2282  maskSpanSetList : `list`
2283  List of SpanSets with artifacts.
2284  """
2285 
2286  maskSpanSetList = []
2287  x0, y0 = epochCountImage.getXY0()
2288  for i, span in enumerate(spanSetList):
2289  y, x = span.indices()
2290  yIdxLocal = [y1 - y0 for y1 in y]
2291  xIdxLocal = [x1 - x0 for x1 in x]
2292  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2293  totalN = nImage.array[yIdxLocal, xIdxLocal]
2294 
2295  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2296  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2297  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2298  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2299  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2300  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2301  (outlierN <= effectiveMaxNumEpochs))
2302  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2303  if percentBelowThreshold > self.config.spatialThreshold:
2304  maskSpanSetList.append(span)
2305 
2306  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2307  # If a candidate is contained by a footprint on the template coadd, do not clip
2308  filteredMaskSpanSetList = []
2309  for span in maskSpanSetList:
2310  doKeep = True
2311  for footprint in footprintsToExclude.positive.getFootprints():
2312  if footprint.spans.contains(span):
2313  doKeep = False
2314  break
2315  if doKeep:
2316  filteredMaskSpanSetList.append(span)
2317  maskSpanSetList = filteredMaskSpanSetList
2318 
2319  return maskSpanSetList
2320 
2321  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2322  """Fetch a warp from the butler and return a warpDiff.
2323 
2324  Parameters
2325  ----------
2326  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2327  Butler dataRef for the warp.
2328  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2329  An image scaler object.
2330  templateCoadd : `lsst.afw.image.Exposure`
2331  Exposure to be substracted from the scaled warp.
2332 
2333  Returns
2334  -------
2335  warp : `lsst.afw.image.Exposure`
2336  Exposure of the image difference between the warp and template.
2337  """
2338 
2339  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2340  warpName = self.getTempExpDatasetName('psfMatched')
2341  if not warpRef.datasetExists(warpName):
2342  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2343  return None
2344  warp = warpRef.get(warpName, immediate=True)
2345  # direct image scaler OK for PSF-matched Warp
2346  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2347  mi = warp.getMaskedImage()
2348  if self.config.doScaleWarpVariance:
2349  try:
2350  self.scaleWarpVariance.run(mi)
2351  except Exception as exc:
2352  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2353  mi -= templateCoadd.getMaskedImage()
2354  return warp
2355 
2356  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2357  """Return a path to which to write debugging output.
2358 
2359  Creates a hyphen-delimited string of dataId values for simple filenames.
2360 
2361  Parameters
2362  ----------
2363  prefix : `str`
2364  Prefix for filename.
2365  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2366  Butler dataRef to make the path from.
2367  coaddLevel : `bool`, optional.
2368  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2369  'filter', but no 'visit').
2370 
2371  Returns
2372  -------
2373  result : `str`
2374  Path for debugging output.
2375  """
2376  if coaddLevel:
2377  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2378  else:
2379  keys = warpRef.dataId.keys()
2380  keyList = sorted(keys, reverse=True)
2381  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2382  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2383  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:149
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:100
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def getTempExpRefList(self, patchRef, calExpRefList)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:249
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:133
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:199
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:113
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def processResults(self, coaddExposure, dataRef)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)