lsst.pipe.tasks  18.1.0-10-g73b8679e+15
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy
24 import warnings
25 import lsst.pex.config as pexConfig
26 import lsst.pex.exceptions as pexExceptions
27 import lsst.geom as geom
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 import lsst.afw.detection as afwDet
33 import lsst.coadd.utils as coaddUtils
34 import lsst.pipe.base as pipeBase
35 import lsst.meas.algorithms as measAlg
36 import lsst.log as log
37 import lsstDebug
38 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo
39 from .interpImage import InterpImageTask
40 from .scaleZeroPoint import ScaleZeroPointTask
41 from .coaddHelpers import groupPatchExposures, getGroupDataRef
42 from .scaleVariance import ScaleVarianceTask
43 from lsst.meas.algorithms import SourceDetectionTask
44 from lsst.pipe.base.shims import ShimButler
45 
46 __all__ = ["AssembleCoaddTask", "AssembleCoaddConfig", "SafeClipAssembleCoaddTask",
47  "SafeClipAssembleCoaddConfig", "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
48 
49 
50 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig):
51  """Configuration parameters for the `AssembleCoaddTask`.
52 
53  Notes
54  -----
55  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
56  only set the bitplane config.brightObjectMaskName. To make this useful you
57  *must* also configure the flags.pixel algorithm, for example by adding
58 
59  .. code-block:: none
60 
61  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
62  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
63 
64  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
65  """
66  warpType = pexConfig.Field(
67  doc="Warp name: one of 'direct' or 'psfMatched'",
68  dtype=str,
69  default="direct",
70  )
71  subregionSize = pexConfig.ListField(
72  dtype=int,
73  doc="Width, height of stack subregion size; "
74  "make small enough that a full stack of images will fit into memory at once.",
75  length=2,
76  default=(2000, 2000),
77  )
78  statistic = pexConfig.Field(
79  dtype=str,
80  doc="Main stacking statistic for aggregating over the epochs.",
81  default="MEANCLIP",
82  )
83  doSigmaClip = pexConfig.Field(
84  dtype=bool,
85  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
86  default=False,
87  )
88  sigmaClip = pexConfig.Field(
89  dtype=float,
90  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
91  default=3.0,
92  )
93  clipIter = pexConfig.Field(
94  dtype=int,
95  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
96  default=2,
97  )
98  calcErrorFromInputVariance = pexConfig.Field(
99  dtype=bool,
100  doc="Calculate coadd variance from input variance by stacking statistic."
101  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
102  default=True,
103  )
104  scaleZeroPoint = pexConfig.ConfigurableField(
105  target=ScaleZeroPointTask,
106  doc="Task to adjust the photometric zero point of the coadd temp exposures",
107  )
108  doInterp = pexConfig.Field(
109  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
110  dtype=bool,
111  default=True,
112  )
113  interpImage = pexConfig.ConfigurableField(
114  target=InterpImageTask,
115  doc="Task to interpolate (and extrapolate) over NaN pixels",
116  )
117  doWrite = pexConfig.Field(
118  doc="Persist coadd?",
119  dtype=bool,
120  default=True,
121  )
122  doNImage = pexConfig.Field(
123  doc="Create image of number of contributing exposures for each pixel",
124  dtype=bool,
125  default=False,
126  )
127  doUsePsfMatchedPolygons = pexConfig.Field(
128  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
129  dtype=bool,
130  default=False,
131  )
132  maskPropagationThresholds = pexConfig.DictField(
133  keytype=str,
134  itemtype=float,
135  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
136  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
137  "would have contributed exceeds this value."),
138  default={"SAT": 0.1},
139  )
140  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
141  doc="Mask planes to remove before coadding")
142  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
143  doc="Set mask and flag bits for bright objects?")
144  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
145  doc="Name of mask bit used for bright objects")
146  coaddPsf = pexConfig.ConfigField(
147  doc="Configuration for CoaddPsf",
148  dtype=measAlg.CoaddPsfConfig,
149  )
150  doAttachTransmissionCurve = pexConfig.Field(
151  dtype=bool, default=False, optional=False,
152  doc=("Attach a piecewise TransmissionCurve for the coadd? "
153  "(requires all input Exposures to have TransmissionCurves).")
154  )
155  inputWarps = pipeBase.InputDatasetField(
156  doc=("Input list of warps to be assemebled i.e. stacked."
157  "WarpType (e.g. direct, psfMatched) is controlled by we warpType config parameter"),
158  nameTemplate="{inputCoaddName}Coadd_{warpType}Warp",
159  storageClass="ExposureF",
160  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
161  manualLoad=True,
162  )
163  skyMap = pipeBase.InputDatasetField(
164  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
165  nameTemplate="{inputCoaddName}Coadd_skyMap",
166  storageClass="SkyMap",
167  dimensions=("skymap", ),
168  scalar=True
169  )
170  brightObjectMask = pipeBase.InputDatasetField(
171  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
172  " BRIGHT_OBJECT."),
173  name="brightObjectMask",
174  storageClass="ObjectMaskCatalog",
175  dimensions=("tract", "patch", "skymap", "abstract_filter"),
176  scalar=True
177  )
178  coaddExposure = pipeBase.OutputDatasetField(
179  doc="Output coadded exposure, produced by stacking input warps",
180  nameTemplate="{outputCoaddName}Coadd",
181  storageClass="ExposureF",
182  dimensions=("tract", "patch", "skymap", "abstract_filter"),
183  scalar=True
184  )
185  nImage = pipeBase.OutputDatasetField(
186  doc="Output image of number of input images per pixel",
187  nameTemplate="{outputCoaddName}Coadd_nImage",
188  storageClass="ImageU",
189  dimensions=("tract", "patch", "skymap", "abstract_filter"),
190  scalar=True
191  )
192 
193  hasFakes = pexConfig.Field(
194  dtype=bool,
195  default=False,
196  doc="Should be set to True if fake sources have been inserted into the input data."
197  )
198 
199  def setDefaults(self):
200  super().setDefaults()
201  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
202  self.formatTemplateNames({"inputCoaddName": "deep", "outputCoaddName": "deep",
203  "warpType": self.warpType})
204  self.quantum.dimensions = ("tract", "patch", "abstract_filter", "skymap")
205 
206  def validate(self):
207  super().validate()
208  if self.doPsfMatch:
209  # Backwards compatibility.
210  # Configs do not have loggers
211  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
212  self.warpType = 'psfMatched'
213  if self.doSigmaClip and self.statistic != "MEANCLIP":
214  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
215  self.statistic = "MEANCLIP"
216  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
217  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
218  "compute and set a non-zero coadd variance estimate." % (self.statistic))
219 
220  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
221  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
222  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
223  if str(k) not in unstackableStats]
224  raise ValueError("statistic %s is not allowed. Please choose one of %s."
225  % (self.statistic, stackableStats))
226 
227 
228 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
229  """Assemble a coadded image from a set of warps (coadded temporary exposures).
230 
231  We want to assemble a coadded image from a set of Warps (also called
232  coadded temporary exposures or ``coaddTempExps``).
233  Each input Warp covers a patch on the sky and corresponds to a single
234  run/visit/exposure of the covered patch. We provide the task with a list
235  of Warps (``selectDataList``) from which it selects Warps that cover the
236  specified patch (pointed at by ``dataRef``).
237  Each Warp that goes into a coadd will typically have an independent
238  photometric zero-point. Therefore, we must scale each Warp to set it to
239  a common photometric zeropoint. WarpType may be one of 'direct' or
240  'psfMatched', and the boolean configs `config.makeDirect` and
241  `config.makePsfMatched` set which of the warp types will be coadded.
242  The coadd is computed as a mean with optional outlier rejection.
243  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
244  Finally, Warps can have bad 'NaN' pixels which received no input from the
245  source calExps. We interpolate over these bad (NaN) pixels.
246 
247  `AssembleCoaddTask` uses several sub-tasks. These are
248 
249  - `ScaleZeroPointTask`
250  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
251  - `InterpImageTask`
252  - interpolate across bad pixels (NaN) in the final coadd
253 
254  You can retarget these subtasks if you wish.
255 
256  Notes
257  -----
258  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
259  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
260  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
261  no debug variables of its own. Some of the subtasks may support debug
262  variables. See the documentation for the subtasks for further information.
263 
264  Examples
265  --------
266  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
267  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
268  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
269  inputs: a data reference to the tract patch and filter to be coadded, and
270  a list of Warps to attempt to coadd. These are specified using ``--id`` and
271  ``--selectId``, respectively:
272 
273  .. code-block:: none
274 
275  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
276  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
277 
278  Only the Warps that cover the specified tract and patch will be coadded.
279  A list of the available optional arguments can be obtained by calling
280  ``assembleCoadd.py`` with the ``--help`` command line argument:
281 
282  .. code-block:: none
283 
284  assembleCoadd.py --help
285 
286  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
287  multi-band processing, we will generate the HSC-I & -R band coadds from
288  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
289  assuming that the lsst stack has been already set up, we must set up the
290  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
291  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
292  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
293  coadds, we must first
294 
295  - processCcd
296  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
297  - makeSkyMap
298  - create a skymap that covers the area of the sky present in the raw exposures
299  - makeCoaddTempExp
300  - warp the individual calibrated exposures to the tangent plane of the coadd
301 
302  We can perform all of these steps by running
303 
304  .. code-block:: none
305 
306  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
307 
308  This will produce warped exposures for each visit. To coadd the warped
309  data, we call assembleCoadd.py as follows:
310 
311  .. code-block:: none
312 
313  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
314  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
315  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
316  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
317  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
318  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
319  --selectId visit=903988 ccd=24
320 
321  that will process the HSC-I band data. The results are written in
322  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
323 
324  You may also choose to run:
325 
326  .. code-block:: none
327 
328  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
329  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
330  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
331  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
332  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
333  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
334  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
335  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
336 
337  to generate the coadd for the HSC-R band if you are interested in
338  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
339  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
340  rather than `AssembleCoaddTask` to make the coadd.
341  """
342  ConfigClass = AssembleCoaddConfig
343  _DefaultName = "assembleCoadd"
344 
345  def __init__(self, *args, **kwargs):
346  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
347  if args:
348  argNames = ["config", "name", "parentTask", "log"]
349  kwargs.update({k: v for k, v in zip(argNames, args)})
350  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
351  "PipelineTask will not take positional args" % argNames, FutureWarning)
352 
353  super().__init__(**kwargs)
354  self.makeSubtask("interpImage")
355  self.makeSubtask("scaleZeroPoint")
356 
357  if self.config.doMaskBrightObjects:
358  mask = afwImage.Mask()
359  try:
360  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
361  except pexExceptions.LsstCppException:
362  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
363  mask.getMaskPlaneDict().keys())
364  del mask
365 
366  self.warpType = self.config.warpType
367 
368  @classmethod
369  def getOutputDatasetTypes(cls, config):
370  """Return output dataset type descriptors
371 
372  Remove output dataset types not produced by the Task
373  """
374  outputTypeDict = super().getOutputDatasetTypes(config)
375  if not config.doNImage:
376  outputTypeDict.pop("nImage", None)
377  return outputTypeDict
378 
379  @classmethod
380  def getInputDatasetTypes(cls, config):
381  """Return input dataset type descriptors
382 
383  Remove input dataset types not used by the Task
384  """
385  inputTypeDict = super().getInputDatasetTypes(config)
386  if not config.doMaskBrightObjects:
387  inputTypeDict.pop("brightObjectMask", None)
388  return inputTypeDict
389 
390  @classmethod
391  def getPrerequisiteDatasetTypes(cls, config):
392  return frozenset(["brightObjectMask"])
393 
394  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
395  """Assemble a coadd from a set of Warps.
396 
397  PipelineTask (Gen3) entry point to Coadd a set of Warps.
398  Analogous to `runDataRef`, it prepares all the data products to be
399  passed to `run`, and processes the results before returning to struct
400  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
401  Therefore, its inputs are accessed subregion by subregion
402  by the `lsst.daf.butler.ShimButler` that quacks like a Gen2
403  `lsst.daf.persistence.Butler`. Updates to this method should
404  correspond to an update in `runDataRef` while both entry points
405  are used.
406 
407  Parameters
408  ----------
409  inputData : `dict`
410  Keys are the names of the configs describing input dataset types.
411  Values are input Python-domain data objects (or lists of objects)
412  retrieved from data butler.
413  inputDataIds : `dict`
414  Keys are the names of the configs describing input dataset types.
415  Values are DataIds (or lists of DataIds) that task consumes for
416  corresponding dataset type.
417  outputDataIds : `dict`
418  Keys are the names of the configs describing input dataset types.
419  Values are DataIds (or lists of DataIds) that task is to produce
420  for corresponding dataset type.
421  butler : `lsst.daf.butler.Butler`
422  Gen3 Butler object for fetching additional data products before
423  running the Task
424 
425  Returns
426  -------
427  result : `lsst.pipe.base.Struct`
428  Result struct with components:
429 
430  - ``coaddExposure`` : coadded exposure (``lsst.afw.image.Exposure``)
431  - ``nImage``: N Image (``lsst.afw.image.Image``)
432  """
433  # Construct skyInfo expected by run
434  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
435  skyMap = inputData["skyMap"]
436  outputDataId = next(iter(outputDataIds.values()))
437  inputData['skyInfo'] = makeSkyInfo(skyMap,
438  tractId=outputDataId['tract'],
439  patchId=outputDataId['patch'])
440 
441  # Construct list of input Shim DataRefs that quack like Gen2 DataRefs
442  butlerShim = ShimButler(butler)
443  warpRefList = [butlerShim.dataRef(self.config.inputWarps.name, dataId=dataId)
444  for dataId in inputDataIds['inputWarps']]
445 
446  # Construct output Shim DataRef
447  patchRef = butlerShim.dataRef(self.config.coaddExposure.name, dataId=outputDataIds['coaddExposure'])
448 
449  # Perform same middle steps as `runDataRef` does
450  inputs = self.prepareInputs(warpRefList)
451  self.log.info("Found %d %s", len(inputs.tempExpRefList),
452  self.getTempExpDatasetName(self.warpType))
453  if len(inputs.tempExpRefList) == 0:
454  self.log.warn("No coadd temporary exposures found")
455  return
456 
457  supplementaryData = self.makeSupplementaryDataGen3(inputData, inputDataIds, outputDataIds, butler)
458 
459  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
460  inputs.weightList, supplementaryData=supplementaryData)
461 
462  self.processResults(retStruct.coaddExposure, patchRef)
463  return retStruct
464 
465  @pipeBase.timeMethod
466  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
467  """Assemble a coadd from a set of Warps.
468 
469  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
470  Compute weights to be applied to each Warp and
471  find scalings to match the photometric zeropoint to a reference Warp.
472  Assemble the Warps using `run`. Interpolate over NaNs and
473  optionally write the coadd to disk. Return the coadded exposure.
474 
475  Parameters
476  ----------
477  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
478  Data reference defining the patch for coaddition and the
479  reference Warp (if ``config.autoReference=False``).
480  Used to access the following data products:
481  - ``self.config.coaddName + "Coadd_skyMap"``
482  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
483  - ``self.config.coaddName + "Coadd"``
484  selectDataList : `list`
485  List of data references to Calexps. Data to be coadded will be
486  selected from this list based on overlap with the patch defined
487  by dataRef, grouped by visit, and converted to a list of data
488  references to warps.
489  warpRefList : `list`
490  List of data references to Warps to be coadded.
491  Note: `warpRefList` is just the new name for `tempExpRefList`.
492 
493  Returns
494  -------
495  retStruct : `lsst.pipe.base.Struct`
496  Result struct with components:
497 
498  - ``coaddExposure``: coadded exposure (``Exposure``).
499  - ``nImage``: exposure count image (``Image``).
500  """
501  if selectDataList and warpRefList:
502  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
503  "and which to use is ambiguous. Please pass only one.")
504 
505  skyInfo = self.getSkyInfo(dataRef)
506  if warpRefList is None:
507  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
508  if len(calExpRefList) == 0:
509  self.log.warn("No exposures to coadd")
510  return
511  self.log.info("Coadding %d exposures", len(calExpRefList))
512 
513  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
514 
515  inputData = self.prepareInputs(warpRefList)
516  self.log.info("Found %d %s", len(inputData.tempExpRefList),
517  self.getTempExpDatasetName(self.warpType))
518  if len(inputData.tempExpRefList) == 0:
519  self.log.warn("No coadd temporary exposures found")
520  return
521 
522  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
523 
524  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
525  inputData.weightList, supplementaryData=supplementaryData)
526 
527  self.processResults(retStruct.coaddExposure, dataRef)
528  if self.config.doWrite:
529  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
530  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
531  else:
532  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
533  self.log.info("Persisting %s" % coaddDatasetName)
534  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
535  if self.config.doNImage and retStruct.nImage is not None:
536  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
537 
538  return retStruct
539 
540  def processResults(self, coaddExposure, dataRef):
541  """Interpolate over missing data and mask bright stars.
542 
543  Parameters
544  ----------
545  coaddExposure : `lsst.afw.image.Exposure`
546  The coadded exposure to process.
547  dataRef : `lsst.daf.persistence.ButlerDataRef`
548  Butler data reference for supplementary data.
549  """
550  if self.config.doInterp:
551  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
552  # The variance must be positive; work around for DM-3201.
553  varArray = coaddExposure.variance.array
554  with numpy.errstate(invalid="ignore"):
555  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
556 
557  if self.config.doMaskBrightObjects:
558  brightObjectMasks = self.readBrightObjectMasks(dataRef)
559  self.setBrightObjectMasks(coaddExposure, dataRef.dataId, brightObjectMasks)
560 
561  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
562  """Make additional inputs to run() specific to subclasses (Gen2)
563 
564  Duplicates interface of `runDataRef` method
565  Available to be implemented by subclasses only if they need the
566  coadd dataRef for performing preliminary processing before
567  assembling the coadd.
568 
569  Parameters
570  ----------
571  dataRef : `lsst.daf.persistence.ButlerDataRef`
572  Butler data reference for supplementary data.
573  selectDataList : `list`
574  List of data references to Warps.
575  """
576  pass
577 
578  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
579  """Make additional inputs to run() specific to subclasses (Gen3)
580 
581  Duplicates interface of`adaptArgsAndRun` method.
582  Available to be implemented by subclasses only if they need the
583  coadd dataRef for performing preliminary processing before
584  assembling the coadd.
585 
586  Parameters
587  ----------
588  inputData : `dict`
589  Keys are the names of the configs describing input dataset types.
590  Values are input Python-domain data objects (or lists of objects)
591  retrieved from data butler.
592  inputDataIds : `dict`
593  Keys are the names of the configs describing input dataset types.
594  Values are DataIds (or lists of DataIds) that task consumes for
595  corresponding dataset type.
596  DataIds are guaranteed to match data objects in ``inputData``.
597  outputDataIds : `dict`
598  Keys are the names of the configs describing input dataset types.
599  Values are DataIds (or lists of DataIds) that task is to produce
600  for corresponding dataset type.
601  butler : `lsst.daf.butler.Butler`
602  Gen3 Butler object for fetching additional data products before
603  running the Task
604 
605  Returns
606  -------
607  result : `lsst.pipe.base.Struct`
608  Contains whatever additional data the subclass's `run` method needs
609  """
610  pass
611 
612  def getTempExpRefList(self, patchRef, calExpRefList):
613  """Generate list data references corresponding to warped exposures
614  that lie within the patch to be coadded.
615 
616  Parameters
617  ----------
618  patchRef : `dataRef`
619  Data reference for patch.
620  calExpRefList : `list`
621  List of data references for input calexps.
622 
623  Returns
624  -------
625  tempExpRefList : `list`
626  List of Warp/CoaddTempExp data references.
627  """
628  butler = patchRef.getButler()
629  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
630  self.getTempExpDatasetName(self.warpType))
631  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
632  g, groupData.keys) for
633  g in groupData.groups.keys()]
634  return tempExpRefList
635 
636  def prepareInputs(self, refList):
637  """Prepare the input warps for coaddition by measuring the weight for
638  each warp and the scaling for the photometric zero point.
639 
640  Each Warp has its own photometric zeropoint and background variance.
641  Before coadding these Warps together, compute a scale factor to
642  normalize the photometric zeropoint and compute the weight for each Warp.
643 
644  Parameters
645  ----------
646  refList : `list`
647  List of data references to tempExp
648 
649  Returns
650  -------
651  result : `lsst.pipe.base.Struct`
652  Result struct with components:
653 
654  - ``tempExprefList``: `list` of data references to tempExp.
655  - ``weightList``: `list` of weightings.
656  - ``imageScalerList``: `list` of image scalers.
657  """
658  statsCtrl = afwMath.StatisticsControl()
659  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
660  statsCtrl.setNumIter(self.config.clipIter)
661  statsCtrl.setAndMask(self.getBadPixelMask())
662  statsCtrl.setNanSafe(True)
663  # compute tempExpRefList: a list of tempExpRef that actually exist
664  # and weightList: a list of the weight of the associated coadd tempExp
665  # and imageScalerList: a list of scale factors for the associated coadd tempExp
666  tempExpRefList = []
667  weightList = []
668  imageScalerList = []
669  tempExpName = self.getTempExpDatasetName(self.warpType)
670  for tempExpRef in refList:
671  if not tempExpRef.datasetExists(tempExpName):
672  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
673  continue
674 
675  tempExp = tempExpRef.get(tempExpName, immediate=True)
676  # Ignore any input warp that is empty of data
677  if numpy.isnan(tempExp.image.array).all():
678  continue
679  maskedImage = tempExp.getMaskedImage()
680  imageScaler = self.scaleZeroPoint.computeImageScaler(
681  exposure=tempExp,
682  dataRef=tempExpRef,
683  )
684  try:
685  imageScaler.scaleMaskedImage(maskedImage)
686  except Exception as e:
687  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
688  continue
689  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
690  afwMath.MEANCLIP, statsCtrl)
691  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
692  weight = 1.0 / float(meanVar)
693  if not numpy.isfinite(weight):
694  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
695  continue
696  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
697 
698  del maskedImage
699  del tempExp
700 
701  tempExpRefList.append(tempExpRef)
702  weightList.append(weight)
703  imageScalerList.append(imageScaler)
704 
705  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
706  imageScalerList=imageScalerList)
707 
708  def prepareStats(self, mask=None):
709  """Prepare the statistics for coadding images.
710 
711  Parameters
712  ----------
713  mask : `int`, optional
714  Bit mask value to exclude from coaddition.
715 
716  Returns
717  -------
718  stats : `lsst.pipe.base.Struct`
719  Statistics structure with the following fields:
720 
721  - ``statsCtrl``: Statistics control object for coadd
722  (`lsst.afw.math.StatisticsControl`)
723  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
724  """
725  if mask is None:
726  mask = self.getBadPixelMask()
727  statsCtrl = afwMath.StatisticsControl()
728  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
729  statsCtrl.setNumIter(self.config.clipIter)
730  statsCtrl.setAndMask(mask)
731  statsCtrl.setNanSafe(True)
732  statsCtrl.setWeighted(True)
733  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
734  for plane, threshold in self.config.maskPropagationThresholds.items():
735  bit = afwImage.Mask.getMaskPlane(plane)
736  statsCtrl.setMaskPropagationThreshold(bit, threshold)
737  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
738  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
739 
740  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
741  altMaskList=None, mask=None, supplementaryData=None):
742  """Assemble a coadd from input warps
743 
744  Assemble the coadd using the provided list of coaddTempExps. Since
745  the full coadd covers a patch (a large area), the assembly is
746  performed over small areas on the image at a time in order to
747  conserve memory usage. Iterate over subregions within the outer
748  bbox of the patch using `assembleSubregion` to stack the corresponding
749  subregions from the coaddTempExps with the statistic specified.
750  Set the edge bits the coadd mask based on the weight map.
751 
752  Parameters
753  ----------
754  skyInfo : `lsst.pipe.base.Struct`
755  Struct with geometric information about the patch.
756  tempExpRefList : `list`
757  List of data references to Warps (previously called CoaddTempExps).
758  imageScalerList : `list`
759  List of image scalers.
760  weightList : `list`
761  List of weights
762  altMaskList : `list`, optional
763  List of alternate masks to use rather than those stored with
764  tempExp.
765  mask : `int`, optional
766  Bit mask value to exclude from coaddition.
767  supplementaryData : lsst.pipe.base.Struct, optional
768  Struct with additional data products needed to assemble coadd.
769  Only used by subclasses that implement `makeSupplementaryData`
770  and override `run`.
771 
772  Returns
773  -------
774  result : `lsst.pipe.base.Struct`
775  Result struct with components:
776 
777  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
778  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
779  """
780  tempExpName = self.getTempExpDatasetName(self.warpType)
781  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
782  stats = self.prepareStats(mask=mask)
783 
784  if altMaskList is None:
785  altMaskList = [None]*len(tempExpRefList)
786 
787  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
788  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
789  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
790  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
791  coaddMaskedImage = coaddExposure.getMaskedImage()
792  subregionSizeArr = self.config.subregionSize
793  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
794  # if nImage is requested, create a zero one which can be passed to assembleSubregion
795  if self.config.doNImage:
796  nImage = afwImage.ImageU(skyInfo.bbox)
797  else:
798  nImage = None
799  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
800  try:
801  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
802  weightList, altMaskList, stats.flags, stats.ctrl,
803  nImage=nImage)
804  except Exception as e:
805  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
806 
807  self.setInexactPsf(coaddMaskedImage.getMask())
808  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
809  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
810  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
811  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
812 
813  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
814  """Set the metadata for the coadd.
815 
816  This basic implementation sets the filter from the first input.
817 
818  Parameters
819  ----------
820  coaddExposure : `lsst.afw.image.Exposure`
821  The target exposure for the coadd.
822  tempExpRefList : `list`
823  List of data references to tempExp.
824  weightList : `list`
825  List of weights.
826  """
827  assert len(tempExpRefList) == len(weightList), "Length mismatch"
828  tempExpName = self.getTempExpDatasetName(self.warpType)
829  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
830  # (and we need more than just the PropertySet that contains the header), which is not possible
831  # with the current butler (see #2777).
832  tempExpList = [tempExpRef.get(tempExpName + "_sub",
833  bbox=geom.Box2I(coaddExposure.getBBox().getMin(),
834  geom.Extent2I(1, 1)), immediate=True)
835  for tempExpRef in tempExpRefList]
836  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
837 
838  coaddExposure.setFilter(tempExpList[0].getFilter())
839  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
840  coaddInputs.ccds.reserve(numCcds)
841  coaddInputs.visits.reserve(len(tempExpList))
842 
843  for tempExp, weight in zip(tempExpList, weightList):
844  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
845 
846  if self.config.doUsePsfMatchedPolygons:
847  self.shrinkValidPolygons(coaddInputs)
848 
849  coaddInputs.visits.sort()
850  if self.warpType == "psfMatched":
851  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
852  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
853  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
854  # having the maximum width (sufficient because square)
855  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
856  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
857  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
858  else:
859  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
860  self.config.coaddPsf.makeControl())
861  coaddExposure.setPsf(psf)
862  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
863  coaddExposure.getWcs())
864  coaddExposure.getInfo().setApCorrMap(apCorrMap)
865  if self.config.doAttachTransmissionCurve:
866  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
867  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
868 
869  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
870  altMaskList, statsFlags, statsCtrl, nImage=None):
871  """Assemble the coadd for a sub-region.
872 
873  For each coaddTempExp, check for (and swap in) an alternative mask
874  if one is passed. Remove mask planes listed in
875  `config.removeMaskPlanes`. Finally, stack the actual exposures using
876  `lsst.afw.math.statisticsStack` with the statistic specified by
877  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
878  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
879  an N-sigma clipped mean where N and iterations are specified by
880  statsCtrl. Assign the stacked subregion back to the coadd.
881 
882  Parameters
883  ----------
884  coaddExposure : `lsst.afw.image.Exposure`
885  The target exposure for the coadd.
886  bbox : `lsst.geom.Box`
887  Sub-region to coadd.
888  tempExpRefList : `list`
889  List of data reference to tempExp.
890  imageScalerList : `list`
891  List of image scalers.
892  weightList : `list`
893  List of weights.
894  altMaskList : `list`
895  List of alternate masks to use rather than those stored with
896  tempExp, or None. Each element is dict with keys = mask plane
897  name to which to add the spans.
898  statsFlags : `lsst.afw.math.Property`
899  Property object for statistic for coadd.
900  statsCtrl : `lsst.afw.math.StatisticsControl`
901  Statistics control object for coadd.
902  nImage : `lsst.afw.image.ImageU`, optional
903  Keeps track of exposure count for each pixel.
904  """
905  self.log.debug("Computing coadd over %s", bbox)
906  tempExpName = self.getTempExpDatasetName(self.warpType)
907  coaddExposure.mask.addMaskPlane("REJECTED")
908  coaddExposure.mask.addMaskPlane("CLIPPED")
909  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
910  maskMap = self.setRejectedMaskMapping(statsCtrl)
911  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
912  maskedImageList = []
913  if nImage is not None:
914  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
915  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
916  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
917  maskedImage = exposure.getMaskedImage()
918  mask = maskedImage.getMask()
919  if altMask is not None:
920  self.applyAltMaskPlanes(mask, altMask)
921  imageScaler.scaleMaskedImage(maskedImage)
922 
923  # Add 1 for each pixel which is not excluded by the exclude mask.
924  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
925  if nImage is not None:
926  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
927  if self.config.removeMaskPlanes:
928  self.removeMaskPlanes(maskedImage)
929  maskedImageList.append(maskedImage)
930 
931  with self.timer("stack"):
932  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
933  clipped, # also set output to CLIPPED if sigma-clipped
934  maskMap)
935  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
936  if nImage is not None:
937  nImage.assign(subNImage, bbox)
938 
939  def removeMaskPlanes(self, maskedImage):
940  """Unset the mask of an image for mask planes specified in the config.
941 
942  Parameters
943  ----------
944  maskedImage : `lsst.afw.image.MaskedImage`
945  The masked image to be modified.
946  """
947  mask = maskedImage.getMask()
948  for maskPlane in self.config.removeMaskPlanes:
949  try:
950  mask &= ~mask.getPlaneBitMask(maskPlane)
951  except pexExceptions.InvalidParameterError:
952  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
953  maskPlane)
954 
955  @staticmethod
956  def setRejectedMaskMapping(statsCtrl):
957  """Map certain mask planes of the warps to new planes for the coadd.
958 
959  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
960  or CLIPPED, set it to REJECTED on the coadd.
961  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
962  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
963 
964  Parameters
965  ----------
966  statsCtrl : `lsst.afw.math.StatisticsControl`
967  Statistics control object for coadd
968 
969  Returns
970  -------
971  maskMap : `list` of `tuple` of `int`
972  A list of mappings of mask planes of the warped exposures to
973  mask planes of the coadd.
974  """
975  edge = afwImage.Mask.getPlaneBitMask("EDGE")
976  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
977  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
978  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
979  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
980  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
981  (clipped, clipped)]
982  return maskMap
983 
984  def applyAltMaskPlanes(self, mask, altMaskSpans):
985  """Apply in place alt mask formatted as SpanSets to a mask.
986 
987  Parameters
988  ----------
989  mask : `lsst.afw.image.Mask`
990  Original mask.
991  altMaskSpans : `dict`
992  SpanSet lists to apply. Each element contains the new mask
993  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
994  and list of SpanSets to apply to the mask.
995 
996  Returns
997  -------
998  mask : `lsst.afw.image.Mask`
999  Updated mask.
1000  """
1001  if self.config.doUsePsfMatchedPolygons:
1002  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1003  # Clear away any other masks outside the validPolygons. These pixels are no longer
1004  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1005  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1006  # This mask-clearing step must only occur *before* applying the new masks below
1007  for spanSet in altMaskSpans['NO_DATA']:
1008  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1009 
1010  for plane, spanSetList in altMaskSpans.items():
1011  maskClipValue = mask.addMaskPlane(plane)
1012  for spanSet in spanSetList:
1013  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1014  return mask
1015 
1016  def shrinkValidPolygons(self, coaddInputs):
1017  """Shrink coaddInputs' ccds' ValidPolygons in place.
1018 
1019  Either modify each ccd's validPolygon in place, or if CoaddInputs
1020  does not have a validPolygon, create one from its bbox.
1021 
1022  Parameters
1023  ----------
1024  coaddInputs : `lsst.afw.image.coaddInputs`
1025  Original mask.
1026 
1027  """
1028  for ccd in coaddInputs.ccds:
1029  polyOrig = ccd.getValidPolygon()
1030  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1031  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1032  if polyOrig:
1033  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1034  else:
1035  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1036  ccd.setValidPolygon(validPolygon)
1037 
1038  def readBrightObjectMasks(self, dataRef):
1039  """Retrieve the bright object masks.
1040 
1041  Returns None on failure.
1042 
1043  Parameters
1044  ----------
1045  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1046  A Butler dataRef.
1047 
1048  Returns
1049  -------
1050  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1051  Bright object mask from the Butler object, or None if it cannot
1052  be retrieved.
1053  """
1054  try:
1055  return dataRef.get("brightObjectMask", immediate=True)
1056  except Exception as e:
1057  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1058  return None
1059 
1060  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
1061  """Set the bright object masks.
1062 
1063  Parameters
1064  ----------
1065  exposure : `lsst.afw.image.Exposure`
1066  Exposure under consideration.
1067  dataId : `lsst.daf.persistence.dataId`
1068  Data identifier dict for patch.
1069  brightObjectMasks : `lsst.afw.table`
1070  Table of bright objects to mask.
1071  """
1072 
1073  if brightObjectMasks is None:
1074  self.log.warn("Unable to apply bright object mask: none supplied")
1075  return
1076  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1077  mask = exposure.getMaskedImage().getMask()
1078  wcs = exposure.getWcs()
1079  plateScale = wcs.getPixelScale().asArcseconds()
1080 
1081  for rec in brightObjectMasks:
1082  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1083  if rec["type"] == "box":
1084  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1085  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1086  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1087 
1088  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1089  bbox = geom.Box2I(center - halfSize, center + halfSize)
1090 
1091  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1092  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1093  spans = afwGeom.SpanSet(bbox)
1094  elif rec["type"] == "circle":
1095  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1096  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1097  else:
1098  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1099  continue
1100  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1101 
1102  def setInexactPsf(self, mask):
1103  """Set INEXACT_PSF mask plane.
1104 
1105  If any of the input images isn't represented in the coadd (due to
1106  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1107  these pixels.
1108 
1109  Parameters
1110  ----------
1111  mask : `lsst.afw.image.Mask`
1112  Coadded exposure's mask, modified in-place.
1113  """
1114  mask.addMaskPlane("INEXACT_PSF")
1115  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1116  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1117  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1118  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1119  array = mask.getArray()
1120  selected = array & (sensorEdge | clipped | rejected) > 0
1121  array[selected] |= inexactPsf
1122 
1123  @classmethod
1124  def _makeArgumentParser(cls):
1125  """Create an argument parser.
1126  """
1127  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1128  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
1129  cls.ConfigClass().warpType + "Warp",
1130  help="data ID, e.g. --id tract=12345 patch=1,2",
1131  ContainerClass=AssembleCoaddDataIdContainer)
1132  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1133  ContainerClass=SelectDataIdContainer)
1134  return parser
1135 
1136  @staticmethod
1137  def _subBBoxIter(bbox, subregionSize):
1138  """Iterate over subregions of a bbox.
1139 
1140  Parameters
1141  ----------
1142  bbox : `lsst.geom.Box2I`
1143  Bounding box over which to iterate.
1144  subregionSize: `lsst.geom.Extent2I`
1145  Size of sub-bboxes.
1146 
1147  Yields
1148  ------
1149  subBBox : `lsst.geom.Box2I`
1150  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1151  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1152  the edges of ``bbox``, but it will never be empty.
1153  """
1154  if bbox.isEmpty():
1155  raise RuntimeError("bbox %s is empty" % (bbox,))
1156  if subregionSize[0] < 1 or subregionSize[1] < 1:
1157  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1158 
1159  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1160  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1161  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1162  subBBox.clip(bbox)
1163  if subBBox.isEmpty():
1164  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1165  "colShift=%s, rowShift=%s" %
1166  (bbox, subregionSize, colShift, rowShift))
1167  yield subBBox
1168 
1169 
1170 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1171  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1172  """
1173 
1174  def makeDataRefList(self, namespace):
1175  """Make self.refList from self.idList.
1176 
1177  Parameters
1178  ----------
1179  namespace
1180  Results of parsing command-line (with ``butler`` and ``log`` elements).
1181  """
1182  datasetType = namespace.config.coaddName + "Coadd"
1183  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1184 
1185  for dataId in self.idList:
1186  # tract and patch are required
1187  for key in keysCoadd:
1188  if key not in dataId:
1189  raise RuntimeError("--id must include " + key)
1190 
1191  dataRef = namespace.butler.dataRef(
1192  datasetType=datasetType,
1193  dataId=dataId,
1194  )
1195  self.refList.append(dataRef)
1196 
1197 
1198 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1199  """Function to count the number of pixels with a specific mask in a
1200  footprint.
1201 
1202  Find the intersection of mask & footprint. Count all pixels in the mask
1203  that are in the intersection that have bitmask set but do not have
1204  ignoreMask set. Return the count.
1205 
1206  Parameters
1207  ----------
1208  mask : `lsst.afw.image.Mask`
1209  Mask to define intersection region by.
1210  footprint : `lsst.afw.detection.Footprint`
1211  Footprint to define the intersection region by.
1212  bitmask
1213  Specific mask that we wish to count the number of occurances of.
1214  ignoreMask
1215  Pixels to not consider.
1216 
1217  Returns
1218  -------
1219  result : `int`
1220  Count of number of pixels in footprint with specified mask.
1221  """
1222  bbox = footprint.getBBox()
1223  bbox.clip(mask.getBBox(afwImage.PARENT))
1224  fp = afwImage.Mask(bbox)
1225  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1226  footprint.spans.setMask(fp, bitmask)
1227  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1228  (subMask.getArray() & ignoreMask) == 0).sum()
1229 
1230 
1232  """Configuration parameters for the SafeClipAssembleCoaddTask.
1233  """
1234  clipDetection = pexConfig.ConfigurableField(
1235  target=SourceDetectionTask,
1236  doc="Detect sources on difference between unclipped and clipped coadd")
1237  minClipFootOverlap = pexConfig.Field(
1238  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1239  dtype=float,
1240  default=0.6
1241  )
1242  minClipFootOverlapSingle = pexConfig.Field(
1243  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1244  "clipped when only one visit overlaps",
1245  dtype=float,
1246  default=0.5
1247  )
1248  minClipFootOverlapDouble = pexConfig.Field(
1249  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1250  "clipped when two visits overlap",
1251  dtype=float,
1252  default=0.45
1253  )
1254  maxClipFootOverlapDouble = pexConfig.Field(
1255  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1256  "considering two visits",
1257  dtype=float,
1258  default=0.15
1259  )
1260  minBigOverlap = pexConfig.Field(
1261  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1262  "when labeling clipped footprints",
1263  dtype=int,
1264  default=100
1265  )
1266 
1267  def setDefaults(self):
1268  """Set default values for clipDetection.
1269 
1270  Notes
1271  -----
1272  The numeric values for these configuration parameters were
1273  empirically determined, future work may further refine them.
1274  """
1275  AssembleCoaddConfig.setDefaults(self)
1276  self.clipDetection.doTempLocalBackground = False
1277  self.clipDetection.reEstimateBackground = False
1278  self.clipDetection.returnOriginalFootprints = False
1279  self.clipDetection.thresholdPolarity = "both"
1280  self.clipDetection.thresholdValue = 2
1281  self.clipDetection.nSigmaToGrow = 2
1282  self.clipDetection.minPixels = 4
1283  self.clipDetection.isotropicGrow = True
1284  self.clipDetection.thresholdType = "pixel_stdev"
1285  self.sigmaClip = 1.5
1286  self.clipIter = 3
1287  self.statistic = "MEAN"
1288 
1289  def validate(self):
1290  if self.doSigmaClip:
1291  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1292  "Ignoring doSigmaClip.")
1293  self.doSigmaClip = False
1294  if self.statistic != "MEAN":
1295  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1296  "(%s chosen). Please set statistic to MEAN."
1297  % (self.statistic))
1298  AssembleCoaddTask.ConfigClass.validate(self)
1299 
1300 
1302  """Assemble a coadded image from a set of coadded temporary exposures,
1303  being careful to clip & flag areas with potential artifacts.
1304 
1305  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1306  we clip outliers). The problem with doing this is that when computing the
1307  coadd PSF at a given location, individual visit PSFs from visits with
1308  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1309  In this task, we correct for this behavior by creating a new
1310  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1311  coaddTempExps and the final coadd where
1312 
1313  i. difference imaging suggests that there is an outlier and
1314  ii. this outlier appears on only one or two images.
1315 
1316  Such regions will not contribute to the final coadd. Furthermore, any
1317  routine to determine the coadd PSF can now be cognizant of clipped regions.
1318  Note that the algorithm implemented by this task is preliminary and works
1319  correctly for HSC data. Parameter modifications and or considerable
1320  redesigning of the algorithm is likley required for other surveys.
1321 
1322  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1323  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1324  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1325  if you wish.
1326 
1327  Notes
1328  -----
1329  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1330  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1331  see `baseDebug` for more about ``debug.py`` files.
1332  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1333  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1334  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1335  for further information.
1336 
1337  Examples
1338  --------
1339  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1340  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1341  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1342 
1343  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1344  and filter to be coadded (specified using
1345  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1346  along with a list of coaddTempExps to attempt to coadd (specified using
1347  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1348  Only the coaddTempExps that cover the specified tract and patch will be
1349  coadded. A list of the available optional arguments can be obtained by
1350  calling assembleCoadd.py with the --help command line argument:
1351 
1352  .. code-block:: none
1353 
1354  assembleCoadd.py --help
1355 
1356  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1357  context of multi-band processing, we will generate the HSC-I & -R band
1358  coadds from HSC engineering test data provided in the ci_hsc package.
1359  To begin, assuming that the lsst stack has been already set up, we must
1360  set up the obs_subaru and ci_hsc packages. This defines the environment
1361  variable $CI_HSC_DIR and points at the location of the package. The raw
1362  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1363  the coadds, we must first
1364 
1365  - ``processCcd``
1366  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1367  - ``makeSkyMap``
1368  create a skymap that covers the area of the sky present in the raw exposures
1369  - ``makeCoaddTempExp``
1370  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1371 
1372  We can perform all of these steps by running
1373 
1374  .. code-block:: none
1375 
1376  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1377 
1378  This will produce warped coaddTempExps for each visit. To coadd the
1379  warped data, we call ``assembleCoadd.py`` as follows:
1380 
1381  .. code-block:: none
1382 
1383  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1384  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1385  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1386  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1387  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1388  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1389  --selectId visit=903988 ccd=24
1390 
1391  This will process the HSC-I band data. The results are written in
1392  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1393 
1394  You may also choose to run:
1395 
1396  .. code-block:: none
1397 
1398  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1399  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1400  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1401  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1402  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1403  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1404  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1405  --selectId visit=903346 ccd=12
1406 
1407  to generate the coadd for the HSC-R band if you are interested in following
1408  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1409  """
1410  ConfigClass = SafeClipAssembleCoaddConfig
1411  _DefaultName = "safeClipAssembleCoadd"
1412 
1413  def __init__(self, *args, **kwargs):
1414  AssembleCoaddTask.__init__(self, *args, **kwargs)
1415  schema = afwTable.SourceTable.makeMinimalSchema()
1416  self.makeSubtask("clipDetection", schema=schema)
1417 
1418  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1419  """Assemble the coadd for a region.
1420 
1421  Compute the difference of coadds created with and without outlier
1422  rejection to identify coadd pixels that have outlier values in some
1423  individual visits.
1424  Detect clipped regions on the difference image and mark these regions
1425  on the one or two individual coaddTempExps where they occur if there
1426  is significant overlap between the clipped region and a source. This
1427  leaves us with a set of footprints from the difference image that have
1428  been identified as having occured on just one or two individual visits.
1429  However, these footprints were generated from a difference image. It
1430  is conceivable for a large diffuse source to have become broken up
1431  into multiple footprints acrosss the coadd difference in this process.
1432  Determine the clipped region from all overlapping footprints from the
1433  detected sources in each visit - these are big footprints.
1434  Combine the small and big clipped footprints and mark them on a new
1435  bad mask plane.
1436  Generate the coadd using `AssembleCoaddTask.run` without outlier
1437  removal. Clipped footprints will no longer make it into the coadd
1438  because they are marked in the new bad mask plane.
1439 
1440  Parameters
1441  ----------
1442  skyInfo : `lsst.pipe.base.Struct`
1443  Patch geometry information, from getSkyInfo
1444  tempExpRefList : `list`
1445  List of data reference to tempExp
1446  imageScalerList : `list`
1447  List of image scalers
1448  weightList : `list`
1449  List of weights
1450 
1451  Returns
1452  -------
1453  result : `lsst.pipe.base.Struct`
1454  Result struct with components:
1455 
1456  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
1457  - ``nImage``: exposure count image (``lsst.afw.image.Image``).
1458 
1459  Notes
1460  -----
1461  args and kwargs are passed but ignored in order to match the call
1462  signature expected by the parent task.
1463  """
1464  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1465  mask = exp.getMaskedImage().getMask()
1466  mask.addMaskPlane("CLIPPED")
1467 
1468  result = self.detectClip(exp, tempExpRefList)
1469 
1470  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1471 
1472  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1473  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1474  # Append big footprints from individual Warps to result.clipSpans
1475  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1476  result.detectionFootprints, maskClipValue, maskDetValue,
1477  exp.getBBox())
1478  # Create mask of the current clipped footprints
1479  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1480  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1481 
1482  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1483  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1484  maskClip |= maskClipBig
1485 
1486  # Assemble coadd from base class, but ignoring CLIPPED pixels
1487  badMaskPlanes = self.config.badMaskPlanes[:]
1488  badMaskPlanes.append("CLIPPED")
1489  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1490  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1491  result.clipSpans, mask=badPixelMask)
1492 
1493  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1494  """Return an exposure that contains the difference between unclipped
1495  and clipped coadds.
1496 
1497  Generate a difference image between clipped and unclipped coadds.
1498  Compute the difference image by subtracting an outlier-clipped coadd
1499  from an outlier-unclipped coadd. Return the difference image.
1500 
1501  Parameters
1502  ----------
1503  skyInfo : `lsst.pipe.base.Struct`
1504  Patch geometry information, from getSkyInfo
1505  tempExpRefList : `list`
1506  List of data reference to tempExp
1507  imageScalerList : `list`
1508  List of image scalers
1509  weightList : `list`
1510  List of weights
1511 
1512  Returns
1513  -------
1514  exp : `lsst.afw.image.Exposure`
1515  Difference image of unclipped and clipped coadd wrapped in an Exposure
1516  """
1517  # Clone and upcast self.config because current self.config is frozen
1518  config = AssembleCoaddConfig()
1519  # getattr necessary because subtasks do not survive Config.toDict()
1520  configIntersection = {k: getattr(self.config, k)
1521  for k, v in self.config.toDict().items() if (k in config.keys())}
1522  config.update(**configIntersection)
1523 
1524  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1525  config.statistic = 'MEAN'
1526  task = AssembleCoaddTask(config=config)
1527  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1528 
1529  config.statistic = 'MEANCLIP'
1530  task = AssembleCoaddTask(config=config)
1531  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1532 
1533  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1534  coaddDiff -= coaddClip.getMaskedImage()
1535  exp = afwImage.ExposureF(coaddDiff)
1536  exp.setPsf(coaddMean.getPsf())
1537  return exp
1538 
1539  def detectClip(self, exp, tempExpRefList):
1540  """Detect clipped regions on an exposure and set the mask on the
1541  individual tempExp masks.
1542 
1543  Detect footprints in the difference image after smoothing the
1544  difference image with a Gaussian kernal. Identify footprints that
1545  overlap with one or two input ``coaddTempExps`` by comparing the
1546  computed overlap fraction to thresholds set in the config. A different
1547  threshold is applied depending on the number of overlapping visits
1548  (restricted to one or two). If the overlap exceeds the thresholds,
1549  the footprint is considered "CLIPPED" and is marked as such on the
1550  coaddTempExp. Return a struct with the clipped footprints, the indices
1551  of the ``coaddTempExps`` that end up overlapping with the clipped
1552  footprints, and a list of new masks for the ``coaddTempExps``.
1553 
1554  Parameters
1555  ----------
1556  exp : `lsst.afw.image.Exposure`
1557  Exposure to run detection on.
1558  tempExpRefList : `list`
1559  List of data reference to tempExp.
1560 
1561  Returns
1562  -------
1563  result : `lsst.pipe.base.Struct`
1564  Result struct with components:
1565 
1566  - ``clipFootprints``: list of clipped footprints.
1567  - ``clipIndices``: indices for each ``clippedFootprint`` in
1568  ``tempExpRefList``.
1569  - ``clipSpans``: List of dictionaries containing spanSet lists
1570  to clip. Each element contains the new maskplane name
1571  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1572  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1573  compressed into footprints.
1574  """
1575  mask = exp.getMaskedImage().getMask()
1576  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1577  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1578  # Merge positive and negative together footprints together
1579  fpSet.positive.merge(fpSet.negative)
1580  footprints = fpSet.positive
1581  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1582  ignoreMask = self.getBadPixelMask()
1583 
1584  clipFootprints = []
1585  clipIndices = []
1586  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1587 
1588  # for use by detectClipBig
1589  visitDetectionFootprints = []
1590 
1591  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1592  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1593  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1594 
1595  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1596  for i, warpRef in enumerate(tempExpRefList):
1597  tmpExpMask = warpRef.get(self.getTempExpDatasetName(self.warpType),
1598  immediate=True).getMaskedImage().getMask()
1599  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1600  afwImage.PARENT, True)
1601  maskVisitDet &= maskDetValue
1602  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1603  visitDetectionFootprints.append(visitFootprints)
1604 
1605  for j, footprint in enumerate(footprints.getFootprints()):
1606  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1607  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1608 
1609  # build a list of clipped spans for each visit
1610  for j, footprint in enumerate(footprints.getFootprints()):
1611  nPixel = footprint.getArea()
1612  overlap = [] # hold the overlap with each visit
1613  indexList = [] # index of visit in global list
1614  for i in range(len(tempExpRefList)):
1615  ignore = ignoreArr[i, j]
1616  overlapDet = overlapDetArr[i, j]
1617  totPixel = nPixel - ignore
1618 
1619  # If we have more bad pixels than detection skip
1620  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1621  continue
1622  overlap.append(overlapDet/float(totPixel))
1623  indexList.append(i)
1624 
1625  overlap = numpy.array(overlap)
1626  if not len(overlap):
1627  continue
1628 
1629  keep = False # Should this footprint be marked as clipped?
1630  keepIndex = [] # Which tempExps does the clipped footprint belong to
1631 
1632  # If footprint only has one overlap use a lower threshold
1633  if len(overlap) == 1:
1634  if overlap[0] > self.config.minClipFootOverlapSingle:
1635  keep = True
1636  keepIndex = [0]
1637  else:
1638  # This is the general case where only visit should be clipped
1639  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1640  if len(clipIndex) == 1:
1641  keep = True
1642  keepIndex = [clipIndex[0]]
1643 
1644  # Test if there are clipped objects that overlap two different visits
1645  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1646  if len(clipIndex) == 2 and len(overlap) > 3:
1647  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1648  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1649  keep = True
1650  keepIndex = clipIndex
1651 
1652  if not keep:
1653  continue
1654 
1655  for index in keepIndex:
1656  globalIndex = indexList[index]
1657  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1658 
1659  clipIndices.append(numpy.array(indexList)[keepIndex])
1660  clipFootprints.append(footprint)
1661 
1662  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1663  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1664 
1665  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1666  maskClipValue, maskDetValue, coaddBBox):
1667  """Return individual warp footprints for large artifacts and append
1668  them to ``clipList`` in place.
1669 
1670  Identify big footprints composed of many sources in the coadd
1671  difference that may have originated in a large diffuse source in the
1672  coadd. We do this by indentifying all clipped footprints that overlap
1673  significantly with each source in all the coaddTempExps.
1674 
1675  Parameters
1676  ----------
1677  clipList : `list`
1678  List of alt mask SpanSets with clipping information. Modified.
1679  clipFootprints : `list`
1680  List of clipped footprints.
1681  clipIndices : `list`
1682  List of which entries in tempExpClipList each footprint belongs to.
1683  maskClipValue
1684  Mask value of clipped pixels.
1685  maskDetValue
1686  Mask value of detected pixels.
1687  coaddBBox : `lsst.geom.Box`
1688  BBox of the coadd and warps.
1689 
1690  Returns
1691  -------
1692  bigFootprintsCoadd : `list`
1693  List of big footprints
1694  """
1695  bigFootprintsCoadd = []
1696  ignoreMask = self.getBadPixelMask()
1697  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1698  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1699  for footprint in visitFootprints.getFootprints():
1700  footprint.spans.setMask(maskVisitDet, maskDetValue)
1701 
1702  # build a mask of clipped footprints that are in this visit
1703  clippedFootprintsVisit = []
1704  for foot, clipIndex in zip(clipFootprints, clipIndices):
1705  if index not in clipIndex:
1706  continue
1707  clippedFootprintsVisit.append(foot)
1708  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1709  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1710 
1711  bigFootprintsVisit = []
1712  for foot in visitFootprints.getFootprints():
1713  if foot.getArea() < self.config.minBigOverlap:
1714  continue
1715  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1716  if nCount > self.config.minBigOverlap:
1717  bigFootprintsVisit.append(foot)
1718  bigFootprintsCoadd.append(foot)
1719 
1720  for footprint in bigFootprintsVisit:
1721  clippedSpans["CLIPPED"].append(footprint.spans)
1722 
1723  return bigFootprintsCoadd
1724 
1725 
1727  assembleStaticSkyModel = pexConfig.ConfigurableField(
1728  target=AssembleCoaddTask,
1729  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1730  " naive/first-iteration model of the static sky.",
1731  )
1732  detect = pexConfig.ConfigurableField(
1733  target=SourceDetectionTask,
1734  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1735  )
1736  detectTemplate = pexConfig.ConfigurableField(
1737  target=SourceDetectionTask,
1738  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1739  )
1740  maxNumEpochs = pexConfig.Field(
1741  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1742  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1743  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1744  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1745  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1746  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1747  "than transient and not masked.",
1748  dtype=int,
1749  default=2
1750  )
1751  maxFractionEpochsLow = pexConfig.RangeField(
1752  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1753  "Effective maxNumEpochs = "
1754  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1755  dtype=float,
1756  default=0.4,
1757  min=0., max=1.,
1758  )
1759  maxFractionEpochsHigh = pexConfig.RangeField(
1760  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1761  "Effective maxNumEpochs = "
1762  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1763  dtype=float,
1764  default=0.03,
1765  min=0., max=1.,
1766  )
1767  spatialThreshold = pexConfig.RangeField(
1768  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1769  "temporal criteria. If 0, clip all. If 1, clip none.",
1770  dtype=float,
1771  default=0.5,
1772  min=0., max=1.,
1773  inclusiveMin=True, inclusiveMax=True
1774  )
1775  doScaleWarpVariance = pexConfig.Field(
1776  doc="Rescale Warp variance plane using empirical noise?",
1777  dtype=bool,
1778  default=True,
1779  )
1780  scaleWarpVariance = pexConfig.ConfigurableField(
1781  target=ScaleVarianceTask,
1782  doc="Rescale variance on warps",
1783  )
1784  doPreserveContainedBySource = pexConfig.Field(
1785  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1786  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1787  dtype=bool,
1788  default=True,
1789  )
1790  doPrefilterArtifacts = pexConfig.Field(
1791  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1792  "because they will be excluded anyway. This prevents them from contributing "
1793  "to the outlier epoch count image and potentially being labeled as persistant."
1794  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1795  dtype=bool,
1796  default=True
1797  )
1798  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1799  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1800  dtype=str,
1801  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1802  )
1803  prefilterArtifactsRatio = pexConfig.Field(
1804  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1805  dtype=float,
1806  default=0.05
1807  )
1808  psfMatchedWarps = pipeBase.InputDatasetField(
1809  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1810  "Only PSF-Matched Warps make sense for image subtraction. "
1811  "Therefore, they must be in the InputDatasetField and made available to the task."),
1812  nameTemplate="{inputCoaddName}Coadd_psfMatchedWarp",
1813  storageClass="ExposureF",
1814  dimensions=("tract", "patch", "skymap", "visit"),
1815  manualLoad=True
1816  )
1817 
1818  def setDefaults(self):
1819  AssembleCoaddConfig.setDefaults(self)
1820  self.statistic = 'MEAN'
1822 
1823  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1824  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1825  if "EDGE" in self.badMaskPlanes:
1826  self.badMaskPlanes.remove('EDGE')
1827  self.removeMaskPlanes.append('EDGE')
1828  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1829  self.assembleStaticSkyModel.warpType = 'psfMatched'
1830  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1831  self.assembleStaticSkyModel.sigmaClip = 2.5
1832  self.assembleStaticSkyModel.clipIter = 3
1833  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1834  self.assembleStaticSkyModel.doWrite = False
1835  self.detect.doTempLocalBackground = False
1836  self.detect.reEstimateBackground = False
1837  self.detect.returnOriginalFootprints = False
1838  self.detect.thresholdPolarity = "both"
1839  self.detect.thresholdValue = 5
1840  self.detect.nSigmaToGrow = 2
1841  self.detect.minPixels = 4
1842  self.detect.isotropicGrow = True
1843  self.detect.thresholdType = "pixel_stdev"
1844  self.detectTemplate.nSigmaToGrow = 2.0
1845  self.detectTemplate.doTempLocalBackground = False
1846  self.detectTemplate.reEstimateBackground = False
1847  self.detectTemplate.returnOriginalFootprints = False
1848 
1849 
1851  """Assemble a compareWarp coadded image from a set of warps
1852  by masking artifacts detected by comparing PSF-matched warps.
1853 
1854  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1855  we clip outliers). The problem with doing this is that when computing the
1856  coadd PSF at a given location, individual visit PSFs from visits with
1857  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1858  In this task, we correct for this behavior by creating a new badMaskPlane
1859  'CLIPPED' which marks pixels in the individual warps suspected to contain
1860  an artifact. We populate this plane on the input warps by comparing
1861  PSF-matched warps with a PSF-matched median coadd which serves as a
1862  model of the static sky. Any group of pixels that deviates from the
1863  PSF-matched template coadd by more than config.detect.threshold sigma,
1864  is an artifact candidate. The candidates are then filtered to remove
1865  variable sources and sources that are difficult to subtract such as
1866  bright stars. This filter is configured using the config parameters
1867  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1868  the maximum fraction of epochs that the deviation can appear in and still
1869  be considered an artifact. The spatialThreshold is the maximum fraction of
1870  pixels in the footprint of the deviation that appear in other epochs
1871  (where other epochs is defined by the temporalThreshold). If the deviant
1872  region meets this criteria of having a significant percentage of pixels
1873  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1874  set in the mask. These regions will not contribute to the final coadd.
1875  Furthermore, any routine to determine the coadd PSF can now be cognizant
1876  of clipped regions. Note that the algorithm implemented by this task is
1877  preliminary and works correctly for HSC data. Parameter modifications and
1878  or considerable redesigning of the algorithm is likley required for other
1879  surveys.
1880 
1881  ``CompareWarpAssembleCoaddTask`` sub-classes
1882  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1883  as a subtask to generate the TemplateCoadd (the model of the static sky).
1884 
1885  Notes
1886  -----
1887  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1888  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1889  ``baseDebug`` for more about ``debug.py`` files.
1890 
1891  This task supports the following debug variables:
1892 
1893  - ``saveCountIm``
1894  If True then save the Epoch Count Image as a fits file in the `figPath`
1895  - ``figPath``
1896  Path to save the debug fits images and figures
1897 
1898  For example, put something like:
1899 
1900  .. code-block:: python
1901 
1902  import lsstDebug
1903  def DebugInfo(name):
1904  di = lsstDebug.getInfo(name)
1905  if name == "lsst.pipe.tasks.assembleCoadd":
1906  di.saveCountIm = True
1907  di.figPath = "/desired/path/to/debugging/output/images"
1908  return di
1909  lsstDebug.Info = DebugInfo
1910 
1911  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1912  ``--debug`` flag. Some subtasks may have their own debug variables;
1913  see individual Task documentation.
1914 
1915  Examples
1916  --------
1917  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1918  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1919  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1920  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1921  and filter to be coadded (specified using
1922  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1923  along with a list of coaddTempExps to attempt to coadd (specified using
1924  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1925  Only the warps that cover the specified tract and patch will be coadded.
1926  A list of the available optional arguments can be obtained by calling
1927  ``assembleCoadd.py`` with the ``--help`` command line argument:
1928 
1929  .. code-block:: none
1930 
1931  assembleCoadd.py --help
1932 
1933  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1934  context of multi-band processing, we will generate the HSC-I & -R band
1935  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1936  To begin, assuming that the lsst stack has been already set up, we must
1937  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1938  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1939  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1940  directory. To begin assembling the coadds, we must first
1941 
1942  - processCcd
1943  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1944  - makeSkyMap
1945  create a skymap that covers the area of the sky present in the raw exposures
1946  - makeCoaddTempExp
1947  warp the individual calibrated exposures to the tangent plane of the coadd
1948 
1949  We can perform all of these steps by running
1950 
1951  .. code-block:: none
1952 
1953  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1954 
1955  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1956  warped data, we call ``assembleCoadd.py`` as follows:
1957 
1958  .. code-block:: none
1959 
1960  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1961  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1962  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1963  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1964  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1965  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1966  --selectId visit=903988 ccd=24
1967 
1968  This will process the HSC-I band data. The results are written in
1969  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1970  """
1971  ConfigClass = CompareWarpAssembleCoaddConfig
1972  _DefaultName = "compareWarpAssembleCoadd"
1973 
1974  def __init__(self, *args, **kwargs):
1975  AssembleCoaddTask.__init__(self, *args, **kwargs)
1976  self.makeSubtask("assembleStaticSkyModel")
1977  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1978  self.makeSubtask("detect", schema=detectionSchema)
1979  if self.config.doPreserveContainedBySource:
1980  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1981  if self.config.doScaleWarpVariance:
1982  self.makeSubtask("scaleWarpVariance")
1983 
1984  def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler):
1985  """Make inputs specific to Subclass with Gen 3 API
1986 
1987  Calls Gen3 `adaptArgsAndRun` instead of the Gen2 specific `runDataRef`
1988 
1989  Duplicates interface of`adaptArgsAndRun` method.
1990  Available to be implemented by subclasses only if they need the
1991  coadd dataRef for performing preliminary processing before
1992  assembling the coadd.
1993 
1994  Parameters
1995  ----------
1996  inputData : `dict`
1997  Keys are the names of the configs describing input dataset types.
1998  Values are input Python-domain data objects (or lists of objects)
1999  retrieved from data butler.
2000  inputDataIds : `dict`
2001  Keys are the names of the configs describing input dataset types.
2002  Values are DataIds (or lists of DataIds) that task consumes for
2003  corresponding dataset type.
2004  DataIds are guaranteed to match data objects in ``inputData``.
2005  outputDataIds : `dict`
2006  Keys are the names of the configs describing input dataset types.
2007  Values are DataIds (or lists of DataIds) that task is to produce
2008  for corresponding dataset type.
2009  butler : `lsst.daf.butler.Butler`
2010  Gen3 Butler object for fetching additional data products before
2011  running the Task
2012 
2013  Returns
2014  -------
2015  result : `lsst.pipe.base.Struct`
2016  Result struct with components:
2017 
2018  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2019  - ``nImage``: N Image (``lsst.afw.image.Image``)
2020 
2021 
2022  """
2023  templateCoadd = self.assembleStaticSkyModel.adaptArgsAndRun(inputData, inputDataIds,
2024  outputDataIds, butler)
2025  if templateCoadd is None:
2026  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2027 
2028  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2029  nImage=templateCoadd.nImage)
2030 
2031  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2032  """Make inputs specific to Subclass.
2033 
2034  Generate a templateCoadd to use as a native model of static sky to
2035  subtract from warps.
2036 
2037  Parameters
2038  ----------
2039  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2040  Butler dataRef for supplementary data.
2041  selectDataList : `list` (optional)
2042  Optional List of data references to Calexps.
2043  warpRefList : `list` (optional)
2044  Optional List of data references to Warps.
2045 
2046  Returns
2047  -------
2048  result : `lsst.pipe.base.Struct`
2049  Result struct with components:
2050 
2051  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2052  - ``nImage``: N Image (``lsst.afw.image.Image``)
2053  """
2054  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2055  if templateCoadd is None:
2056  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2057 
2058  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2059  nImage=templateCoadd.nImage)
2060 
2061  def _noTemplateMessage(self, warpType):
2062  warpName = (warpType[0].upper() + warpType[1:])
2063  message = """No %(warpName)s warps were found to build the template coadd which is
2064  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2065  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2066  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2067 
2068  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2069  another algorithm like:
2070 
2071  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2072  config.assemble.retarget(SafeClipAssembleCoaddTask)
2073  """ % {"warpName": warpName}
2074  return message
2075 
2076  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2077  supplementaryData, *args, **kwargs):
2078  """Assemble the coadd.
2079 
2080  Find artifacts and apply them to the warps' masks creating a list of
2081  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2082  plane. Then pass these alternative masks to the base class's `run`
2083  method.
2084 
2085  Parameters
2086  ----------
2087  skyInfo : `lsst.pipe.base.Struct`
2088  Patch geometry information.
2089  tempExpRefList : `list`
2090  List of data references to warps.
2091  imageScalerList : `list`
2092  List of image scalers.
2093  weightList : `list`
2094  List of weights.
2095  supplementaryData : `lsst.pipe.base.Struct`
2096  This Struct must contain a ``templateCoadd`` that serves as the
2097  model of the static sky.
2098 
2099  Returns
2100  -------
2101  result : `lsst.pipe.base.Struct`
2102  Result struct with components:
2103 
2104  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
2105  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
2106  """
2107  templateCoadd = supplementaryData.templateCoadd
2108  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
2109  badMaskPlanes = self.config.badMaskPlanes[:]
2110  badMaskPlanes.append("CLIPPED")
2111  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2112 
2113  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2114  spanSetMaskList, mask=badPixelMask)
2115 
2116  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2117  # Psf-Matching moves the real edge inwards
2118  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2119  return result
2120 
2121  def applyAltEdgeMask(self, mask, altMaskList):
2122  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2123 
2124  Parameters
2125  ----------
2126  mask : `lsst.afw.image.Mask`
2127  Original mask.
2128  altMaskList : `list`
2129  List of Dicts containing ``spanSet`` lists.
2130  Each element contains the new mask plane name (e.g. "CLIPPED
2131  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2132  the mask.
2133  """
2134  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2135  for visitMask in altMaskList:
2136  if "EDGE" in visitMask:
2137  for spanSet in visitMask['EDGE']:
2138  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2139 
2140  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2141  """Find artifacts.
2142 
2143  Loop through warps twice. The first loop builds a map with the count
2144  of how many epochs each pixel deviates from the templateCoadd by more
2145  than ``config.chiThreshold`` sigma. The second loop takes each
2146  difference image and filters the artifacts detected in each using
2147  count map to filter out variable sources and sources that are
2148  difficult to subtract cleanly.
2149 
2150  Parameters
2151  ----------
2152  templateCoadd : `lsst.afw.image.Exposure`
2153  Exposure to serve as model of static sky.
2154  tempExpRefList : `list`
2155  List of data references to warps.
2156  imageScalerList : `list`
2157  List of image scalers.
2158 
2159  Returns
2160  -------
2161  altMasks : `list`
2162  List of dicts containing information about CLIPPED
2163  (i.e., artifacts), NO_DATA, and EDGE pixels.
2164  """
2165 
2166  self.log.debug("Generating Count Image, and mask lists.")
2167  coaddBBox = templateCoadd.getBBox()
2168  slateIm = afwImage.ImageU(coaddBBox)
2169  epochCountImage = afwImage.ImageU(coaddBBox)
2170  nImage = afwImage.ImageU(coaddBBox)
2171  spanSetArtifactList = []
2172  spanSetNoDataMaskList = []
2173  spanSetEdgeList = []
2174  badPixelMask = self.getBadPixelMask()
2175 
2176  # mask of the warp diffs should = that of only the warp
2177  templateCoadd.mask.clearAllMaskPlanes()
2178 
2179  if self.config.doPreserveContainedBySource:
2180  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2181  else:
2182  templateFootprints = None
2183 
2184  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2185  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2186  if warpDiffExp is not None:
2187  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2188  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2189  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2190  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2191  fpSet.positive.merge(fpSet.negative)
2192  footprints = fpSet.positive
2193  slateIm.set(0)
2194  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2195 
2196  # Remove artifacts due to defects before they contribute to the epochCountImage
2197  if self.config.doPrefilterArtifacts:
2198  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2199  for spans in spanSetList:
2200  spans.setImage(slateIm, 1, doClip=True)
2201  epochCountImage += slateIm
2202 
2203  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2204  # undergo a second convolution. Pixels with data in the direct warp
2205  # but not in the PSF-matched warp will not have their artifacts detected.
2206  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2207  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2208  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2209  nansMask.setXY0(warpDiffExp.getXY0())
2210  edgeMask = warpDiffExp.mask
2211  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2212  edgeMask.getPlaneBitMask("EDGE")).split()
2213  else:
2214  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2215  # In this case, mask the whole epoch
2216  nansMask = afwImage.MaskX(coaddBBox, 1)
2217  spanSetList = []
2218  spanSetEdgeMask = []
2219 
2220  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2221 
2222  spanSetNoDataMaskList.append(spanSetNoDataMask)
2223  spanSetArtifactList.append(spanSetList)
2224  spanSetEdgeList.append(spanSetEdgeMask)
2225 
2226  if lsstDebug.Info(__name__).saveCountIm:
2227  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2228  epochCountImage.writeFits(path)
2229 
2230  for i, spanSetList in enumerate(spanSetArtifactList):
2231  if spanSetList:
2232  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2233  templateFootprints)
2234  spanSetArtifactList[i] = filteredSpanSetList
2235 
2236  altMasks = []
2237  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2238  altMasks.append({'CLIPPED': artifacts,
2239  'NO_DATA': noData,
2240  'EDGE': edge})
2241  return altMasks
2242 
2243  def prefilterArtifacts(self, spanSetList, exp):
2244  """Remove artifact candidates covered by bad mask plane.
2245 
2246  Any future editing of the candidate list that does not depend on
2247  temporal information should go in this method.
2248 
2249  Parameters
2250  ----------
2251  spanSetList : `list`
2252  List of SpanSets representing artifact candidates.
2253  exp : `lsst.afw.image.Exposure`
2254  Exposure containing mask planes used to prefilter.
2255 
2256  Returns
2257  -------
2258  returnSpanSetList : `list`
2259  List of SpanSets with artifacts.
2260  """
2261  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2262  goodArr = (exp.mask.array & badPixelMask) == 0
2263  returnSpanSetList = []
2264  bbox = exp.getBBox()
2265  x0, y0 = exp.getXY0()
2266  for i, span in enumerate(spanSetList):
2267  y, x = span.clippedTo(bbox).indices()
2268  yIndexLocal = numpy.array(y) - y0
2269  xIndexLocal = numpy.array(x) - x0
2270  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2271  if goodRatio > self.config.prefilterArtifactsRatio:
2272  returnSpanSetList.append(span)
2273  return returnSpanSetList
2274 
2275  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2276  """Filter artifact candidates.
2277 
2278  Parameters
2279  ----------
2280  spanSetList : `list`
2281  List of SpanSets representing artifact candidates.
2282  epochCountImage : `lsst.afw.image.Image`
2283  Image of accumulated number of warpDiff detections.
2284  nImage : `lsst.afw.image.Image`
2285  Image of the accumulated number of total epochs contributing.
2286 
2287  Returns
2288  -------
2289  maskSpanSetList : `list`
2290  List of SpanSets with artifacts.
2291  """
2292 
2293  maskSpanSetList = []
2294  x0, y0 = epochCountImage.getXY0()
2295  for i, span in enumerate(spanSetList):
2296  y, x = span.indices()
2297  yIdxLocal = [y1 - y0 for y1 in y]
2298  xIdxLocal = [x1 - x0 for x1 in x]
2299  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2300  totalN = nImage.array[yIdxLocal, xIdxLocal]
2301 
2302  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2303  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2304  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2305  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2306  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2307  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2308  (outlierN <= effectiveMaxNumEpochs))
2309  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2310  if percentBelowThreshold > self.config.spatialThreshold:
2311  maskSpanSetList.append(span)
2312 
2313  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2314  # If a candidate is contained by a footprint on the template coadd, do not clip
2315  filteredMaskSpanSetList = []
2316  for span in maskSpanSetList:
2317  doKeep = True
2318  for footprint in footprintsToExclude.positive.getFootprints():
2319  if footprint.spans.contains(span):
2320  doKeep = False
2321  break
2322  if doKeep:
2323  filteredMaskSpanSetList.append(span)
2324  maskSpanSetList = filteredMaskSpanSetList
2325 
2326  return maskSpanSetList
2327 
2328  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2329  """Fetch a warp from the butler and return a warpDiff.
2330 
2331  Parameters
2332  ----------
2333  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2334  Butler dataRef for the warp.
2335  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2336  An image scaler object.
2337  templateCoadd : `lsst.afw.image.Exposure`
2338  Exposure to be substracted from the scaled warp.
2339 
2340  Returns
2341  -------
2342  warp : `lsst.afw.image.Exposure`
2343  Exposure of the image difference between the warp and template.
2344  """
2345 
2346  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2347  warpName = self.getTempExpDatasetName('psfMatched')
2348  if not warpRef.datasetExists(warpName):
2349  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2350  return None
2351  warp = warpRef.get(warpName, immediate=True)
2352  # direct image scaler OK for PSF-matched Warp
2353  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2354  mi = warp.getMaskedImage()
2355  if self.config.doScaleWarpVariance:
2356  try:
2357  self.scaleWarpVariance.run(mi)
2358  except Exception as exc:
2359  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2360  mi -= templateCoadd.getMaskedImage()
2361  return warp
2362 
2363  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2364  """Return a path to which to write debugging output.
2365 
2366  Creates a hyphen-delimited string of dataId values for simple filenames.
2367 
2368  Parameters
2369  ----------
2370  prefix : `str`
2371  Prefix for filename.
2372  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2373  Butler dataRef to make the path from.
2374  coaddLevel : `bool`, optional.
2375  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2376  'filter', but no 'visit').
2377 
2378  Returns
2379  -------
2380  result : `str`
2381  Path for debugging output.
2382  """
2383  if coaddLevel:
2384  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2385  else:
2386  keys = warpRef.dataId.keys()
2387  keyList = sorted(keys, reverse=True)
2388  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2389  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2390  return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:150
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
Base class for coaddition.
Definition: coaddBase.py:101
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def getTempExpRefList(self, patchRef, calExpRefList)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:250
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:134
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:165
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:200
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:114
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def processResults(self, coaddExposure, dataRef)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)