lsst.pipe.tasks  21.0.0-38-g070523fc+2384b0eea2
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from lsst.meas.algorithms import SourceDetectionTask
48 from lsst.daf.butler import DeferredDatasetHandle
49 
50 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
51  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
52  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
53 
54 
55 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
56  dimensions=("tract", "patch", "band", "skymap"),
57  defaultTemplates={"inputCoaddName": "deep",
58  "outputCoaddName": "deep",
59  "warpType": "direct",
60  "warpTypeSuffix": "",
61  "fakesType": ""}):
62  inputWarps = pipeBase.connectionTypes.Input(
63  doc=("Input list of warps to be assemebled i.e. stacked."
64  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
65  name="{inputCoaddName}Coadd_{warpType}Warp",
66  storageClass="ExposureF",
67  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
68  deferLoad=True,
69  multiple=True
70  )
71  skyMap = pipeBase.connectionTypes.Input(
72  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
73  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
74  storageClass="SkyMap",
75  dimensions=("skymap", ),
76  )
77  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
78  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
79  " BRIGHT_OBJECT."),
80  name="brightObjectMask",
81  storageClass="ObjectMaskCatalog",
82  dimensions=("tract", "patch", "skymap", "band"),
83  )
84  coaddExposure = pipeBase.connectionTypes.Output(
85  doc="Output coadded exposure, produced by stacking input warps",
86  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
87  storageClass="ExposureF",
88  dimensions=("tract", "patch", "skymap", "band"),
89  )
90  nImage = pipeBase.connectionTypes.Output(
91  doc="Output image of number of input images per pixel",
92  name="{outputCoaddName}Coadd_nImage",
93  storageClass="ImageU",
94  dimensions=("tract", "patch", "skymap", "band"),
95  )
96 
97  def __init__(self, *, config=None):
98  super().__init__(config=config)
99 
100  # Override the connection's name template with config to replicate Gen2 behavior
101  # This duplicates some of the logic in the base class, due to wanting Gen2 and
102  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
103  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
104  templateValues['warpType'] = config.warpType
105  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
106  if config.hasFakes:
107  templateValues['fakesType'] = "_fakes"
108  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
109  for name in self.allConnections}
110  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
111  # End code to remove after deprecation
112 
113  if not config.doMaskBrightObjects:
114  self.prerequisiteInputs.remove("brightObjectMask")
115 
116  if not config.doNImage:
117  self.outputs.remove("nImage")
118 
119 
120 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
121  pipelineConnections=AssembleCoaddConnections):
122  """Configuration parameters for the `AssembleCoaddTask`.
123 
124  Notes
125  -----
126  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
127  only set the bitplane config.brightObjectMaskName. To make this useful you
128  *must* also configure the flags.pixel algorithm, for example by adding
129 
130  .. code-block:: none
131 
132  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
133  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
134 
135  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
136  """
137  warpType = pexConfig.Field(
138  doc="Warp name: one of 'direct' or 'psfMatched'",
139  dtype=str,
140  default="direct",
141  )
142  subregionSize = pexConfig.ListField(
143  dtype=int,
144  doc="Width, height of stack subregion size; "
145  "make small enough that a full stack of images will fit into memory at once.",
146  length=2,
147  default=(2000, 2000),
148  )
149  statistic = pexConfig.Field(
150  dtype=str,
151  doc="Main stacking statistic for aggregating over the epochs.",
152  default="MEANCLIP",
153  )
154  doSigmaClip = pexConfig.Field(
155  dtype=bool,
156  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
157  default=False,
158  )
159  sigmaClip = pexConfig.Field(
160  dtype=float,
161  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
162  default=3.0,
163  )
164  clipIter = pexConfig.Field(
165  dtype=int,
166  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
167  default=2,
168  )
169  calcErrorFromInputVariance = pexConfig.Field(
170  dtype=bool,
171  doc="Calculate coadd variance from input variance by stacking statistic."
172  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
173  default=True,
174  )
175  scaleZeroPoint = pexConfig.ConfigurableField(
176  target=ScaleZeroPointTask,
177  doc="Task to adjust the photometric zero point of the coadd temp exposures",
178  )
179  doInterp = pexConfig.Field(
180  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
181  dtype=bool,
182  default=True,
183  )
184  interpImage = pexConfig.ConfigurableField(
185  target=InterpImageTask,
186  doc="Task to interpolate (and extrapolate) over NaN pixels",
187  )
188  doWrite = pexConfig.Field(
189  doc="Persist coadd?",
190  dtype=bool,
191  default=True,
192  )
193  doNImage = pexConfig.Field(
194  doc="Create image of number of contributing exposures for each pixel",
195  dtype=bool,
196  default=False,
197  )
198  doUsePsfMatchedPolygons = pexConfig.Field(
199  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
200  dtype=bool,
201  default=False,
202  )
203  maskPropagationThresholds = pexConfig.DictField(
204  keytype=str,
205  itemtype=float,
206  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
207  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
208  "would have contributed exceeds this value."),
209  default={"SAT": 0.1},
210  )
211  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
212  doc="Mask planes to remove before coadding")
213  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
214  doc="Set mask and flag bits for bright objects?")
215  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
216  doc="Name of mask bit used for bright objects")
217  coaddPsf = pexConfig.ConfigField(
218  doc="Configuration for CoaddPsf",
219  dtype=measAlg.CoaddPsfConfig,
220  )
221  doAttachTransmissionCurve = pexConfig.Field(
222  dtype=bool, default=False, optional=False,
223  doc=("Attach a piecewise TransmissionCurve for the coadd? "
224  "(requires all input Exposures to have TransmissionCurves).")
225  )
226  hasFakes = pexConfig.Field(
227  dtype=bool,
228  default=False,
229  doc="Should be set to True if fake sources have been inserted into the input data."
230  )
231 
232  def setDefaults(self):
233  super().setDefaults()
234  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
235 
236  def validate(self):
237  super().validate()
238  if self.doPsfMatch:
239  # Backwards compatibility.
240  # Configs do not have loggers
241  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
242  self.warpType = 'psfMatched'
243  if self.doSigmaClip and self.statistic != "MEANCLIP":
244  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
245  self.statistic = "MEANCLIP"
246  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
247  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
248  "compute and set a non-zero coadd variance estimate." % (self.statistic))
249 
250  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
251  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
252  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
253  if str(k) not in unstackableStats]
254  raise ValueError("statistic %s is not allowed. Please choose one of %s."
255  % (self.statistic, stackableStats))
256 
257 
258 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
259  """Assemble a coadded image from a set of warps (coadded temporary exposures).
260 
261  We want to assemble a coadded image from a set of Warps (also called
262  coadded temporary exposures or ``coaddTempExps``).
263  Each input Warp covers a patch on the sky and corresponds to a single
264  run/visit/exposure of the covered patch. We provide the task with a list
265  of Warps (``selectDataList``) from which it selects Warps that cover the
266  specified patch (pointed at by ``dataRef``).
267  Each Warp that goes into a coadd will typically have an independent
268  photometric zero-point. Therefore, we must scale each Warp to set it to
269  a common photometric zeropoint. WarpType may be one of 'direct' or
270  'psfMatched', and the boolean configs `config.makeDirect` and
271  `config.makePsfMatched` set which of the warp types will be coadded.
272  The coadd is computed as a mean with optional outlier rejection.
273  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
274  Finally, Warps can have bad 'NaN' pixels which received no input from the
275  source calExps. We interpolate over these bad (NaN) pixels.
276 
277  `AssembleCoaddTask` uses several sub-tasks. These are
278 
279  - `ScaleZeroPointTask`
280  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
281  - `InterpImageTask`
282  - interpolate across bad pixels (NaN) in the final coadd
283 
284  You can retarget these subtasks if you wish.
285 
286  Notes
287  -----
288  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
289  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
290  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
291  no debug variables of its own. Some of the subtasks may support debug
292  variables. See the documentation for the subtasks for further information.
293 
294  Examples
295  --------
296  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
297  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
298  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
299  inputs: a data reference to the tract patch and filter to be coadded, and
300  a list of Warps to attempt to coadd. These are specified using ``--id`` and
301  ``--selectId``, respectively:
302 
303  .. code-block:: none
304 
305  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
306  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
307 
308  Only the Warps that cover the specified tract and patch will be coadded.
309  A list of the available optional arguments can be obtained by calling
310  ``assembleCoadd.py`` with the ``--help`` command line argument:
311 
312  .. code-block:: none
313 
314  assembleCoadd.py --help
315 
316  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
317  multi-band processing, we will generate the HSC-I & -R band coadds from
318  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
319  assuming that the lsst stack has been already set up, we must set up the
320  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
321  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
322  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
323  coadds, we must first
324 
325  - processCcd
326  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
327  - makeSkyMap
328  - create a skymap that covers the area of the sky present in the raw exposures
329  - makeCoaddTempExp
330  - warp the individual calibrated exposures to the tangent plane of the coadd
331 
332  We can perform all of these steps by running
333 
334  .. code-block:: none
335 
336  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
337 
338  This will produce warped exposures for each visit. To coadd the warped
339  data, we call assembleCoadd.py as follows:
340 
341  .. code-block:: none
342 
343  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
344  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
345  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
346  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
347  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
348  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
349  --selectId visit=903988 ccd=24
350 
351  that will process the HSC-I band data. The results are written in
352  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
353 
354  You may also choose to run:
355 
356  .. code-block:: none
357 
358  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
359  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
360  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
361  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
362  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
363  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
364  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
365  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
366 
367  to generate the coadd for the HSC-R band if you are interested in
368  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
369  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
370  rather than `AssembleCoaddTask` to make the coadd.
371  """
372  ConfigClass = AssembleCoaddConfig
373  _DefaultName = "assembleCoadd"
374 
375  def __init__(self, *args, **kwargs):
376  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
377  if args:
378  argNames = ["config", "name", "parentTask", "log"]
379  kwargs.update({k: v for k, v in zip(argNames, args)})
380  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
381  "PipelineTask will not take positional args" % argNames, FutureWarning)
382 
383  super().__init__(**kwargs)
384  self.makeSubtask("interpImage")
385  self.makeSubtask("scaleZeroPoint")
386 
387  if self.config.doMaskBrightObjects:
388  mask = afwImage.Mask()
389  try:
390  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
391  except pexExceptions.LsstCppException:
392  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
393  mask.getMaskPlaneDict().keys())
394  del mask
395 
396  self.warpType = self.config.warpType
397 
398  @utils.inheritDoc(pipeBase.PipelineTask)
399  def runQuantum(self, butlerQC, inputRefs, outputRefs):
400  # Docstring to be formatted with info from PipelineTask.runQuantum
401  """
402  Notes
403  -----
404  Assemble a coadd from a set of Warps.
405 
406  PipelineTask (Gen3) entry point to Coadd a set of Warps.
407  Analogous to `runDataRef`, it prepares all the data products to be
408  passed to `run`, and processes the results before returning a struct
409  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
410  Therefore, its inputs are accessed subregion by subregion
411  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
412  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
413  correspond to an update in `runDataRef` while both entry points
414  are used.
415  """
416  inputData = butlerQC.get(inputRefs)
417 
418  # Construct skyInfo expected by run
419  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
420  skyMap = inputData["skyMap"]
421  outputDataId = butlerQC.quantum.dataId
422 
423  inputData['skyInfo'] = makeSkyInfo(skyMap,
424  tractId=outputDataId['tract'],
425  patchId=outputDataId['patch'])
426 
427  # Construct list of input Deferred Datasets
428  # These quack a bit like like Gen2 DataRefs
429  warpRefList = inputData['inputWarps']
430  # Perform same middle steps as `runDataRef` does
431  inputs = self.prepareInputs(warpRefList)
432  self.log.info("Found %d %s", len(inputs.tempExpRefList),
433  self.getTempExpDatasetName(self.warpType))
434  if len(inputs.tempExpRefList) == 0:
435  self.log.warn("No coadd temporary exposures found")
436  return
437 
438  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
439  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
440  inputs.weightList, supplementaryData=supplementaryData)
441 
442  inputData.setdefault('brightObjectMask', None)
443  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
444 
445  if self.config.doWrite:
446  butlerQC.put(retStruct, outputRefs)
447  return retStruct
448 
449  @pipeBase.timeMethod
450  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
451  """Assemble a coadd from a set of Warps.
452 
453  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
454  Compute weights to be applied to each Warp and
455  find scalings to match the photometric zeropoint to a reference Warp.
456  Assemble the Warps using `run`. Interpolate over NaNs and
457  optionally write the coadd to disk. Return the coadded exposure.
458 
459  Parameters
460  ----------
461  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
462  Data reference defining the patch for coaddition and the
463  reference Warp (if ``config.autoReference=False``).
464  Used to access the following data products:
465  - ``self.config.coaddName + "Coadd_skyMap"``
466  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
467  - ``self.config.coaddName + "Coadd"``
468  selectDataList : `list`
469  List of data references to Calexps. Data to be coadded will be
470  selected from this list based on overlap with the patch defined
471  by dataRef, grouped by visit, and converted to a list of data
472  references to warps.
473  warpRefList : `list`
474  List of data references to Warps to be coadded.
475  Note: `warpRefList` is just the new name for `tempExpRefList`.
476 
477  Returns
478  -------
479  retStruct : `lsst.pipe.base.Struct`
480  Result struct with components:
481 
482  - ``coaddExposure``: coadded exposure (``Exposure``).
483  - ``nImage``: exposure count image (``Image``).
484  """
485  if selectDataList and warpRefList:
486  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
487  "and which to use is ambiguous. Please pass only one.")
488 
489  skyInfo = self.getSkyInfo(dataRef)
490  if warpRefList is None:
491  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
492  if len(calExpRefList) == 0:
493  self.log.warn("No exposures to coadd")
494  return
495  self.log.info("Coadding %d exposures", len(calExpRefList))
496 
497  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
498 
499  inputData = self.prepareInputs(warpRefList)
500  self.log.info("Found %d %s", len(inputData.tempExpRefList),
501  self.getTempExpDatasetName(self.warpType))
502  if len(inputData.tempExpRefList) == 0:
503  self.log.warn("No coadd temporary exposures found")
504  return
505 
506  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
507 
508  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
509  inputData.weightList, supplementaryData=supplementaryData)
510 
511  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
512  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
513 
514  if self.config.doWrite:
515  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
516  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
517  else:
518  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
519  self.log.info("Persisting %s" % coaddDatasetName)
520  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
521  if self.config.doNImage and retStruct.nImage is not None:
522  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
523 
524  return retStruct
525 
526  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
527  """Interpolate over missing data and mask bright stars.
528 
529  Parameters
530  ----------
531  coaddExposure : `lsst.afw.image.Exposure`
532  The coadded exposure to process.
533  dataRef : `lsst.daf.persistence.ButlerDataRef`
534  Butler data reference for supplementary data.
535  """
536  if self.config.doInterp:
537  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
538  # The variance must be positive; work around for DM-3201.
539  varArray = coaddExposure.variance.array
540  with numpy.errstate(invalid="ignore"):
541  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
542 
543  if self.config.doMaskBrightObjects:
544  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
545 
546  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
547  """Make additional inputs to run() specific to subclasses (Gen2)
548 
549  Duplicates interface of `runDataRef` method
550  Available to be implemented by subclasses only if they need the
551  coadd dataRef for performing preliminary processing before
552  assembling the coadd.
553 
554  Parameters
555  ----------
556  dataRef : `lsst.daf.persistence.ButlerDataRef`
557  Butler data reference for supplementary data.
558  selectDataList : `list` (optional)
559  Optional List of data references to Calexps.
560  warpRefList : `list` (optional)
561  Optional List of data references to Warps.
562  """
563  return pipeBase.Struct()
564 
565  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
566  """Make additional inputs to run() specific to subclasses (Gen3)
567 
568  Duplicates interface of `runQuantum` method.
569  Available to be implemented by subclasses only if they need the
570  coadd dataRef for performing preliminary processing before
571  assembling the coadd.
572 
573  Parameters
574  ----------
575  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
576  Gen3 Butler object for fetching additional data products before
577  running the Task specialized for quantum being processed
578  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
579  Attributes are the names of the connections describing input dataset types.
580  Values are DatasetRefs that task consumes for corresponding dataset type.
581  DataIds are guaranteed to match data objects in ``inputData``.
582  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
583  Attributes are the names of the connections describing output dataset types.
584  Values are DatasetRefs that task is to produce
585  for corresponding dataset type.
586  """
587  return pipeBase.Struct()
588 
589  def getTempExpRefList(self, patchRef, calExpRefList):
590  """Generate list data references corresponding to warped exposures
591  that lie within the patch to be coadded.
592 
593  Parameters
594  ----------
595  patchRef : `dataRef`
596  Data reference for patch.
597  calExpRefList : `list`
598  List of data references for input calexps.
599 
600  Returns
601  -------
602  tempExpRefList : `list`
603  List of Warp/CoaddTempExp data references.
604  """
605  butler = patchRef.getButler()
606  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
607  self.getTempExpDatasetName(self.warpType))
608  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
609  g, groupData.keys) for
610  g in groupData.groups.keys()]
611  return tempExpRefList
612 
613  def prepareInputs(self, refList):
614  """Prepare the input warps for coaddition by measuring the weight for
615  each warp and the scaling for the photometric zero point.
616 
617  Each Warp has its own photometric zeropoint and background variance.
618  Before coadding these Warps together, compute a scale factor to
619  normalize the photometric zeropoint and compute the weight for each Warp.
620 
621  Parameters
622  ----------
623  refList : `list`
624  List of data references to tempExp
625 
626  Returns
627  -------
628  result : `lsst.pipe.base.Struct`
629  Result struct with components:
630 
631  - ``tempExprefList``: `list` of data references to tempExp.
632  - ``weightList``: `list` of weightings.
633  - ``imageScalerList``: `list` of image scalers.
634  """
635  statsCtrl = afwMath.StatisticsControl()
636  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
637  statsCtrl.setNumIter(self.config.clipIter)
638  statsCtrl.setAndMask(self.getBadPixelMask())
639  statsCtrl.setNanSafe(True)
640  # compute tempExpRefList: a list of tempExpRef that actually exist
641  # and weightList: a list of the weight of the associated coadd tempExp
642  # and imageScalerList: a list of scale factors for the associated coadd tempExp
643  tempExpRefList = []
644  weightList = []
645  imageScalerList = []
646  tempExpName = self.getTempExpDatasetName(self.warpType)
647  for tempExpRef in refList:
648  # Gen3's DeferredDatasetHandles are guaranteed to exist and
649  # therefore have no datasetExists() method
650  if not isinstance(tempExpRef, DeferredDatasetHandle):
651  if not tempExpRef.datasetExists(tempExpName):
652  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
653  continue
654 
655  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
656  # Ignore any input warp that is empty of data
657  if numpy.isnan(tempExp.image.array).all():
658  continue
659  maskedImage = tempExp.getMaskedImage()
660  imageScaler = self.scaleZeroPoint.computeImageScaler(
661  exposure=tempExp,
662  dataRef=tempExpRef,
663  )
664  try:
665  imageScaler.scaleMaskedImage(maskedImage)
666  except Exception as e:
667  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
668  continue
669  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
670  afwMath.MEANCLIP, statsCtrl)
671  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
672  weight = 1.0 / float(meanVar)
673  if not numpy.isfinite(weight):
674  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
675  continue
676  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
677 
678  del maskedImage
679  del tempExp
680 
681  tempExpRefList.append(tempExpRef)
682  weightList.append(weight)
683  imageScalerList.append(imageScaler)
684 
685  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
686  imageScalerList=imageScalerList)
687 
688  def prepareStats(self, mask=None):
689  """Prepare the statistics for coadding images.
690 
691  Parameters
692  ----------
693  mask : `int`, optional
694  Bit mask value to exclude from coaddition.
695 
696  Returns
697  -------
698  stats : `lsst.pipe.base.Struct`
699  Statistics structure with the following fields:
700 
701  - ``statsCtrl``: Statistics control object for coadd
702  (`lsst.afw.math.StatisticsControl`)
703  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
704  """
705  if mask is None:
706  mask = self.getBadPixelMask()
707  statsCtrl = afwMath.StatisticsControl()
708  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
709  statsCtrl.setNumIter(self.config.clipIter)
710  statsCtrl.setAndMask(mask)
711  statsCtrl.setNanSafe(True)
712  statsCtrl.setWeighted(True)
713  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
714  for plane, threshold in self.config.maskPropagationThresholds.items():
715  bit = afwImage.Mask.getMaskPlane(plane)
716  statsCtrl.setMaskPropagationThreshold(bit, threshold)
717  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
718  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
719 
720  @pipeBase.timeMethod
721  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
722  altMaskList=None, mask=None, supplementaryData=None):
723  """Assemble a coadd from input warps
724 
725  Assemble the coadd using the provided list of coaddTempExps. Since
726  the full coadd covers a patch (a large area), the assembly is
727  performed over small areas on the image at a time in order to
728  conserve memory usage. Iterate over subregions within the outer
729  bbox of the patch using `assembleSubregion` to stack the corresponding
730  subregions from the coaddTempExps with the statistic specified.
731  Set the edge bits the coadd mask based on the weight map.
732 
733  Parameters
734  ----------
735  skyInfo : `lsst.pipe.base.Struct`
736  Struct with geometric information about the patch.
737  tempExpRefList : `list`
738  List of data references to Warps (previously called CoaddTempExps).
739  imageScalerList : `list`
740  List of image scalers.
741  weightList : `list`
742  List of weights
743  altMaskList : `list`, optional
744  List of alternate masks to use rather than those stored with
745  tempExp.
746  mask : `int`, optional
747  Bit mask value to exclude from coaddition.
748  supplementaryData : lsst.pipe.base.Struct, optional
749  Struct with additional data products needed to assemble coadd.
750  Only used by subclasses that implement `makeSupplementaryData`
751  and override `run`.
752 
753  Returns
754  -------
755  result : `lsst.pipe.base.Struct`
756  Result struct with components:
757 
758  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
759  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
760  - ``warpRefList``: input list of refs to the warps (
761  ``lsst.daf.butler.DeferredDatasetHandle`` or
762  ``lsst.daf.persistence.ButlerDataRef``)
763  (unmodified)
764  - ``imageScalerList``: input list of image scalers (unmodified)
765  - ``weightList``: input list of weights (unmodified)
766  """
767  tempExpName = self.getTempExpDatasetName(self.warpType)
768  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
769  stats = self.prepareStats(mask=mask)
770 
771  if altMaskList is None:
772  altMaskList = [None]*len(tempExpRefList)
773 
774  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
775  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
776  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
777  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
778  coaddMaskedImage = coaddExposure.getMaskedImage()
779  subregionSizeArr = self.config.subregionSize
780  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
781  # if nImage is requested, create a zero one which can be passed to assembleSubregion
782  if self.config.doNImage:
783  nImage = afwImage.ImageU(skyInfo.bbox)
784  else:
785  nImage = None
786  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
787  try:
788  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
789  weightList, altMaskList, stats.flags, stats.ctrl,
790  nImage=nImage)
791  except Exception as e:
792  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
793 
794  self.setInexactPsf(coaddMaskedImage.getMask())
795  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
796  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
797  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
798  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
799  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
800  weightList=weightList)
801 
802  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
803  """Set the metadata for the coadd.
804 
805  This basic implementation sets the filter from the first input.
806 
807  Parameters
808  ----------
809  coaddExposure : `lsst.afw.image.Exposure`
810  The target exposure for the coadd.
811  tempExpRefList : `list`
812  List of data references to tempExp.
813  weightList : `list`
814  List of weights.
815  """
816  assert len(tempExpRefList) == len(weightList), "Length mismatch"
817  tempExpName = self.getTempExpDatasetName(self.warpType)
818  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
819  # (and we need more than just the PropertySet that contains the header), which is not possible
820  # with the current butler (see #2777).
821  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
822 
823  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
824  # Gen 3 API
825  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
826  else:
827  # Gen 2 API. Delete this when Gen 2 retired
828  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
829  for tempExpRef in tempExpRefList]
830  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
831 
832  # Set the coadd FilterLabel to the band of the first input exposure:
833  # Coadds are calibrated, so the physical label is now meaningless.
834  coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
835  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
836  coaddInputs.ccds.reserve(numCcds)
837  coaddInputs.visits.reserve(len(tempExpList))
838 
839  for tempExp, weight in zip(tempExpList, weightList):
840  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
841 
842  if self.config.doUsePsfMatchedPolygons:
843  self.shrinkValidPolygons(coaddInputs)
844 
845  coaddInputs.visits.sort()
846  if self.warpType == "psfMatched":
847  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
848  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
849  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
850  # having the maximum width (sufficient because square)
851  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
852  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
853  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
854  else:
855  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
856  self.config.coaddPsf.makeControl())
857  coaddExposure.setPsf(psf)
858  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
859  coaddExposure.getWcs())
860  coaddExposure.getInfo().setApCorrMap(apCorrMap)
861  if self.config.doAttachTransmissionCurve:
862  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
863  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
864 
865  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
866  altMaskList, statsFlags, statsCtrl, nImage=None):
867  """Assemble the coadd for a sub-region.
868 
869  For each coaddTempExp, check for (and swap in) an alternative mask
870  if one is passed. Remove mask planes listed in
871  `config.removeMaskPlanes`. Finally, stack the actual exposures using
872  `lsst.afw.math.statisticsStack` with the statistic specified by
873  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
874  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
875  an N-sigma clipped mean where N and iterations are specified by
876  statsCtrl. Assign the stacked subregion back to the coadd.
877 
878  Parameters
879  ----------
880  coaddExposure : `lsst.afw.image.Exposure`
881  The target exposure for the coadd.
882  bbox : `lsst.geom.Box`
883  Sub-region to coadd.
884  tempExpRefList : `list`
885  List of data reference to tempExp.
886  imageScalerList : `list`
887  List of image scalers.
888  weightList : `list`
889  List of weights.
890  altMaskList : `list`
891  List of alternate masks to use rather than those stored with
892  tempExp, or None. Each element is dict with keys = mask plane
893  name to which to add the spans.
894  statsFlags : `lsst.afw.math.Property`
895  Property object for statistic for coadd.
896  statsCtrl : `lsst.afw.math.StatisticsControl`
897  Statistics control object for coadd.
898  nImage : `lsst.afw.image.ImageU`, optional
899  Keeps track of exposure count for each pixel.
900  """
901  self.log.debug("Computing coadd over %s", bbox)
902  tempExpName = self.getTempExpDatasetName(self.warpType)
903  coaddExposure.mask.addMaskPlane("REJECTED")
904  coaddExposure.mask.addMaskPlane("CLIPPED")
905  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
906  maskMap = self.setRejectedMaskMapping(statsCtrl)
907  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
908  maskedImageList = []
909  if nImage is not None:
910  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
911  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
912 
913  if isinstance(tempExpRef, DeferredDatasetHandle):
914  # Gen 3 API
915  exposure = tempExpRef.get(parameters={'bbox': bbox})
916  else:
917  # Gen 2 API. Delete this when Gen 2 retired
918  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
919 
920  maskedImage = exposure.getMaskedImage()
921  mask = maskedImage.getMask()
922  if altMask is not None:
923  self.applyAltMaskPlanes(mask, altMask)
924  imageScaler.scaleMaskedImage(maskedImage)
925 
926  # Add 1 for each pixel which is not excluded by the exclude mask.
927  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
928  if nImage is not None:
929  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
930  if self.config.removeMaskPlanes:
931  self.removeMaskPlanes(maskedImage)
932  maskedImageList.append(maskedImage)
933 
934  with self.timer("stack"):
935  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
936  clipped, # also set output to CLIPPED if sigma-clipped
937  maskMap)
938  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
939  if nImage is not None:
940  nImage.assign(subNImage, bbox)
941 
942  def removeMaskPlanes(self, maskedImage):
943  """Unset the mask of an image for mask planes specified in the config.
944 
945  Parameters
946  ----------
947  maskedImage : `lsst.afw.image.MaskedImage`
948  The masked image to be modified.
949  """
950  mask = maskedImage.getMask()
951  for maskPlane in self.config.removeMaskPlanes:
952  try:
953  mask &= ~mask.getPlaneBitMask(maskPlane)
954  except pexExceptions.InvalidParameterError:
955  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
956  maskPlane)
957 
958  @staticmethod
959  def setRejectedMaskMapping(statsCtrl):
960  """Map certain mask planes of the warps to new planes for the coadd.
961 
962  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
963  or CLIPPED, set it to REJECTED on the coadd.
964  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
965  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
966 
967  Parameters
968  ----------
969  statsCtrl : `lsst.afw.math.StatisticsControl`
970  Statistics control object for coadd
971 
972  Returns
973  -------
974  maskMap : `list` of `tuple` of `int`
975  A list of mappings of mask planes of the warped exposures to
976  mask planes of the coadd.
977  """
978  edge = afwImage.Mask.getPlaneBitMask("EDGE")
979  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
980  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
981  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
982  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
983  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
984  (clipped, clipped)]
985  return maskMap
986 
987  def applyAltMaskPlanes(self, mask, altMaskSpans):
988  """Apply in place alt mask formatted as SpanSets to a mask.
989 
990  Parameters
991  ----------
992  mask : `lsst.afw.image.Mask`
993  Original mask.
994  altMaskSpans : `dict`
995  SpanSet lists to apply. Each element contains the new mask
996  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
997  and list of SpanSets to apply to the mask.
998 
999  Returns
1000  -------
1001  mask : `lsst.afw.image.Mask`
1002  Updated mask.
1003  """
1004  if self.config.doUsePsfMatchedPolygons:
1005  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1006  # Clear away any other masks outside the validPolygons. These pixels are no longer
1007  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1008  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1009  # This mask-clearing step must only occur *before* applying the new masks below
1010  for spanSet in altMaskSpans['NO_DATA']:
1011  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1012 
1013  for plane, spanSetList in altMaskSpans.items():
1014  maskClipValue = mask.addMaskPlane(plane)
1015  for spanSet in spanSetList:
1016  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1017  return mask
1018 
1019  def shrinkValidPolygons(self, coaddInputs):
1020  """Shrink coaddInputs' ccds' ValidPolygons in place.
1021 
1022  Either modify each ccd's validPolygon in place, or if CoaddInputs
1023  does not have a validPolygon, create one from its bbox.
1024 
1025  Parameters
1026  ----------
1027  coaddInputs : `lsst.afw.image.coaddInputs`
1028  Original mask.
1029 
1030  """
1031  for ccd in coaddInputs.ccds:
1032  polyOrig = ccd.getValidPolygon()
1033  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1034  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1035  if polyOrig:
1036  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1037  else:
1038  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1039  ccd.setValidPolygon(validPolygon)
1040 
1041  def readBrightObjectMasks(self, dataRef):
1042  """Retrieve the bright object masks.
1043 
1044  Returns None on failure.
1045 
1046  Parameters
1047  ----------
1048  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1049  A Butler dataRef.
1050 
1051  Returns
1052  -------
1053  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1054  Bright object mask from the Butler object, or None if it cannot
1055  be retrieved.
1056  """
1057  try:
1058  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1059  except Exception as e:
1060  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1061  return None
1062 
1063  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1064  """Set the bright object masks.
1065 
1066  Parameters
1067  ----------
1068  exposure : `lsst.afw.image.Exposure`
1069  Exposure under consideration.
1070  dataId : `lsst.daf.persistence.dataId`
1071  Data identifier dict for patch.
1072  brightObjectMasks : `lsst.afw.table`
1073  Table of bright objects to mask.
1074  """
1075 
1076  if brightObjectMasks is None:
1077  self.log.warn("Unable to apply bright object mask: none supplied")
1078  return
1079  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1080  mask = exposure.getMaskedImage().getMask()
1081  wcs = exposure.getWcs()
1082  plateScale = wcs.getPixelScale().asArcseconds()
1083 
1084  for rec in brightObjectMasks:
1085  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1086  if rec["type"] == "box":
1087  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1088  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1089  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1090 
1091  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1092  bbox = geom.Box2I(center - halfSize, center + halfSize)
1093 
1094  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1095  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1096  spans = afwGeom.SpanSet(bbox)
1097  elif rec["type"] == "circle":
1098  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1099  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1100  else:
1101  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1102  continue
1103  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1104 
1105  def setInexactPsf(self, mask):
1106  """Set INEXACT_PSF mask plane.
1107 
1108  If any of the input images isn't represented in the coadd (due to
1109  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1110  these pixels.
1111 
1112  Parameters
1113  ----------
1114  mask : `lsst.afw.image.Mask`
1115  Coadded exposure's mask, modified in-place.
1116  """
1117  mask.addMaskPlane("INEXACT_PSF")
1118  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1119  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1120  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1121  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1122  array = mask.getArray()
1123  selected = array & (sensorEdge | clipped | rejected) > 0
1124  array[selected] |= inexactPsf
1125 
1126  @classmethod
1127  def _makeArgumentParser(cls):
1128  """Create an argument parser.
1129  """
1130  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1131  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1132  + cls.ConfigClass().warpType + "Warp",
1133  help="data ID, e.g. --id tract=12345 patch=1,2",
1134  ContainerClass=AssembleCoaddDataIdContainer)
1135  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1136  ContainerClass=SelectDataIdContainer)
1137  return parser
1138 
1139  @staticmethod
1140  def _subBBoxIter(bbox, subregionSize):
1141  """Iterate over subregions of a bbox.
1142 
1143  Parameters
1144  ----------
1145  bbox : `lsst.geom.Box2I`
1146  Bounding box over which to iterate.
1147  subregionSize: `lsst.geom.Extent2I`
1148  Size of sub-bboxes.
1149 
1150  Yields
1151  ------
1152  subBBox : `lsst.geom.Box2I`
1153  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1154  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1155  the edges of ``bbox``, but it will never be empty.
1156  """
1157  if bbox.isEmpty():
1158  raise RuntimeError("bbox %s is empty" % (bbox,))
1159  if subregionSize[0] < 1 or subregionSize[1] < 1:
1160  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1161 
1162  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1163  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1164  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1165  subBBox.clip(bbox)
1166  if subBBox.isEmpty():
1167  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1168  "colShift=%s, rowShift=%s" %
1169  (bbox, subregionSize, colShift, rowShift))
1170  yield subBBox
1171 
1172 
1173 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1174  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1175  """
1176 
1177  def makeDataRefList(self, namespace):
1178  """Make self.refList from self.idList.
1179 
1180  Parameters
1181  ----------
1182  namespace
1183  Results of parsing command-line (with ``butler`` and ``log`` elements).
1184  """
1185  datasetType = namespace.config.coaddName + "Coadd"
1186  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1187 
1188  for dataId in self.idList:
1189  # tract and patch are required
1190  for key in keysCoadd:
1191  if key not in dataId:
1192  raise RuntimeError("--id must include " + key)
1193 
1194  dataRef = namespace.butler.dataRef(
1195  datasetType=datasetType,
1196  dataId=dataId,
1197  )
1198  self.refList.append(dataRef)
1199 
1200 
1201 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1202  """Function to count the number of pixels with a specific mask in a
1203  footprint.
1204 
1205  Find the intersection of mask & footprint. Count all pixels in the mask
1206  that are in the intersection that have bitmask set but do not have
1207  ignoreMask set. Return the count.
1208 
1209  Parameters
1210  ----------
1211  mask : `lsst.afw.image.Mask`
1212  Mask to define intersection region by.
1213  footprint : `lsst.afw.detection.Footprint`
1214  Footprint to define the intersection region by.
1215  bitmask
1216  Specific mask that we wish to count the number of occurances of.
1217  ignoreMask
1218  Pixels to not consider.
1219 
1220  Returns
1221  -------
1222  result : `int`
1223  Count of number of pixels in footprint with specified mask.
1224  """
1225  bbox = footprint.getBBox()
1226  bbox.clip(mask.getBBox(afwImage.PARENT))
1227  fp = afwImage.Mask(bbox)
1228  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1229  footprint.spans.setMask(fp, bitmask)
1230  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1231  (subMask.getArray() & ignoreMask) == 0).sum()
1232 
1233 
1234 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1235  """Configuration parameters for the SafeClipAssembleCoaddTask.
1236  """
1237  clipDetection = pexConfig.ConfigurableField(
1238  target=SourceDetectionTask,
1239  doc="Detect sources on difference between unclipped and clipped coadd")
1240  minClipFootOverlap = pexConfig.Field(
1241  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1242  dtype=float,
1243  default=0.6
1244  )
1245  minClipFootOverlapSingle = pexConfig.Field(
1246  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1247  "clipped when only one visit overlaps",
1248  dtype=float,
1249  default=0.5
1250  )
1251  minClipFootOverlapDouble = pexConfig.Field(
1252  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1253  "clipped when two visits overlap",
1254  dtype=float,
1255  default=0.45
1256  )
1257  maxClipFootOverlapDouble = pexConfig.Field(
1258  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1259  "considering two visits",
1260  dtype=float,
1261  default=0.15
1262  )
1263  minBigOverlap = pexConfig.Field(
1264  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1265  "when labeling clipped footprints",
1266  dtype=int,
1267  default=100
1268  )
1269 
1270  def setDefaults(self):
1271  """Set default values for clipDetection.
1272 
1273  Notes
1274  -----
1275  The numeric values for these configuration parameters were
1276  empirically determined, future work may further refine them.
1277  """
1278  AssembleCoaddConfig.setDefaults(self)
1279  self.clipDetectionclipDetection.doTempLocalBackground = False
1280  self.clipDetectionclipDetection.reEstimateBackground = False
1281  self.clipDetectionclipDetection.returnOriginalFootprints = False
1282  self.clipDetectionclipDetection.thresholdPolarity = "both"
1283  self.clipDetectionclipDetection.thresholdValue = 2
1284  self.clipDetectionclipDetection.nSigmaToGrow = 2
1285  self.clipDetectionclipDetection.minPixels = 4
1286  self.clipDetectionclipDetection.isotropicGrow = True
1287  self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1288  self.sigmaClipsigmaClip = 1.5
1289  self.clipIterclipIter = 3
1290  self.statisticstatistic = "MEAN"
1291 
1292  def validate(self):
1293  if self.doSigmaClipdoSigmaClip:
1294  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1295  "Ignoring doSigmaClip.")
1296  self.doSigmaClipdoSigmaClip = False
1297  if self.statisticstatistic != "MEAN":
1298  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1299  "(%s chosen). Please set statistic to MEAN."
1300  % (self.statisticstatistic))
1301  AssembleCoaddTask.ConfigClass.validate(self)
1302 
1303 
1304 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1305  """Assemble a coadded image from a set of coadded temporary exposures,
1306  being careful to clip & flag areas with potential artifacts.
1307 
1308  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1309  we clip outliers). The problem with doing this is that when computing the
1310  coadd PSF at a given location, individual visit PSFs from visits with
1311  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1312  In this task, we correct for this behavior by creating a new
1313  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1314  coaddTempExps and the final coadd where
1315 
1316  i. difference imaging suggests that there is an outlier and
1317  ii. this outlier appears on only one or two images.
1318 
1319  Such regions will not contribute to the final coadd. Furthermore, any
1320  routine to determine the coadd PSF can now be cognizant of clipped regions.
1321  Note that the algorithm implemented by this task is preliminary and works
1322  correctly for HSC data. Parameter modifications and or considerable
1323  redesigning of the algorithm is likley required for other surveys.
1324 
1325  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1326  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1327  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1328  if you wish.
1329 
1330  Notes
1331  -----
1332  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1333  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1334  see `baseDebug` for more about ``debug.py`` files.
1335  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1336  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1337  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1338  for further information.
1339 
1340  Examples
1341  --------
1342  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1343  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1344  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1345 
1346  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1347  and filter to be coadded (specified using
1348  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1349  along with a list of coaddTempExps to attempt to coadd (specified using
1350  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1351  Only the coaddTempExps that cover the specified tract and patch will be
1352  coadded. A list of the available optional arguments can be obtained by
1353  calling assembleCoadd.py with the --help command line argument:
1354 
1355  .. code-block:: none
1356 
1357  assembleCoadd.py --help
1358 
1359  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1360  context of multi-band processing, we will generate the HSC-I & -R band
1361  coadds from HSC engineering test data provided in the ci_hsc package.
1362  To begin, assuming that the lsst stack has been already set up, we must
1363  set up the obs_subaru and ci_hsc packages. This defines the environment
1364  variable $CI_HSC_DIR and points at the location of the package. The raw
1365  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1366  the coadds, we must first
1367 
1368  - ``processCcd``
1369  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1370  - ``makeSkyMap``
1371  create a skymap that covers the area of the sky present in the raw exposures
1372  - ``makeCoaddTempExp``
1373  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1374 
1375  We can perform all of these steps by running
1376 
1377  .. code-block:: none
1378 
1379  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1380 
1381  This will produce warped coaddTempExps for each visit. To coadd the
1382  warped data, we call ``assembleCoadd.py`` as follows:
1383 
1384  .. code-block:: none
1385 
1386  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1387  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1388  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1389  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1390  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1391  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1392  --selectId visit=903988 ccd=24
1393 
1394  This will process the HSC-I band data. The results are written in
1395  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1396 
1397  You may also choose to run:
1398 
1399  .. code-block:: none
1400 
1401  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1402  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1403  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1404  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1405  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1406  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1407  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1408  --selectId visit=903346 ccd=12
1409 
1410  to generate the coadd for the HSC-R band if you are interested in following
1411  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1412  """
1413  ConfigClass = SafeClipAssembleCoaddConfig
1414  _DefaultName = "safeClipAssembleCoadd"
1415 
1416  def __init__(self, *args, **kwargs):
1417  AssembleCoaddTask.__init__(self, *args, **kwargs)
1418  schema = afwTable.SourceTable.makeMinimalSchema()
1419  self.makeSubtask("clipDetection", schema=schema)
1420 
1421  @utils.inheritDoc(AssembleCoaddTask)
1422  @pipeBase.timeMethod
1423  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1424  """Assemble the coadd for a region.
1425 
1426  Compute the difference of coadds created with and without outlier
1427  rejection to identify coadd pixels that have outlier values in some
1428  individual visits.
1429  Detect clipped regions on the difference image and mark these regions
1430  on the one or two individual coaddTempExps where they occur if there
1431  is significant overlap between the clipped region and a source. This
1432  leaves us with a set of footprints from the difference image that have
1433  been identified as having occured on just one or two individual visits.
1434  However, these footprints were generated from a difference image. It
1435  is conceivable for a large diffuse source to have become broken up
1436  into multiple footprints acrosss the coadd difference in this process.
1437  Determine the clipped region from all overlapping footprints from the
1438  detected sources in each visit - these are big footprints.
1439  Combine the small and big clipped footprints and mark them on a new
1440  bad mask plane.
1441  Generate the coadd using `AssembleCoaddTask.run` without outlier
1442  removal. Clipped footprints will no longer make it into the coadd
1443  because they are marked in the new bad mask plane.
1444 
1445  Notes
1446  -----
1447  args and kwargs are passed but ignored in order to match the call
1448  signature expected by the parent task.
1449  """
1450  exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1451  mask = exp.getMaskedImage().getMask()
1452  mask.addMaskPlane("CLIPPED")
1453 
1454  result = self.detectClipdetectClip(exp, tempExpRefList)
1455 
1456  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1457 
1458  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1459  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1460  # Append big footprints from individual Warps to result.clipSpans
1461  bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1462  result.detectionFootprints, maskClipValue, maskDetValue,
1463  exp.getBBox())
1464  # Create mask of the current clipped footprints
1465  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1466  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1467 
1468  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1469  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1470  maskClip |= maskClipBig
1471 
1472  # Assemble coadd from base class, but ignoring CLIPPED pixels
1473  badMaskPlanes = self.config.badMaskPlanes[:]
1474  badMaskPlanes.append("CLIPPED")
1475  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1476  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1477  result.clipSpans, mask=badPixelMask)
1478 
1479  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1480  """Return an exposure that contains the difference between unclipped
1481  and clipped coadds.
1482 
1483  Generate a difference image between clipped and unclipped coadds.
1484  Compute the difference image by subtracting an outlier-clipped coadd
1485  from an outlier-unclipped coadd. Return the difference image.
1486 
1487  Parameters
1488  ----------
1489  skyInfo : `lsst.pipe.base.Struct`
1490  Patch geometry information, from getSkyInfo
1491  tempExpRefList : `list`
1492  List of data reference to tempExp
1493  imageScalerList : `list`
1494  List of image scalers
1495  weightList : `list`
1496  List of weights
1497 
1498  Returns
1499  -------
1500  exp : `lsst.afw.image.Exposure`
1501  Difference image of unclipped and clipped coadd wrapped in an Exposure
1502  """
1503  config = AssembleCoaddConfig()
1504  # getattr necessary because subtasks do not survive Config.toDict()
1505  # exclude connections because the class of self.config.connections is not
1506  # the same as AssembleCoaddConfig.connections, and the connections are not
1507  # needed to run this task anyway.
1508  configIntersection = {k: getattr(self.config, k)
1509  for k, v in self.config.toDict().items()
1510  if (k in config.keys() and k != "connections")}
1511  config.update(**configIntersection)
1512 
1513  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1514  config.statistic = 'MEAN'
1515  task = AssembleCoaddTask(config=config)
1516  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1517 
1518  config.statistic = 'MEANCLIP'
1519  task = AssembleCoaddTask(config=config)
1520  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1521 
1522  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1523  coaddDiff -= coaddClip.getMaskedImage()
1524  exp = afwImage.ExposureF(coaddDiff)
1525  exp.setPsf(coaddMean.getPsf())
1526  return exp
1527 
1528  def detectClip(self, exp, tempExpRefList):
1529  """Detect clipped regions on an exposure and set the mask on the
1530  individual tempExp masks.
1531 
1532  Detect footprints in the difference image after smoothing the
1533  difference image with a Gaussian kernal. Identify footprints that
1534  overlap with one or two input ``coaddTempExps`` by comparing the
1535  computed overlap fraction to thresholds set in the config. A different
1536  threshold is applied depending on the number of overlapping visits
1537  (restricted to one or two). If the overlap exceeds the thresholds,
1538  the footprint is considered "CLIPPED" and is marked as such on the
1539  coaddTempExp. Return a struct with the clipped footprints, the indices
1540  of the ``coaddTempExps`` that end up overlapping with the clipped
1541  footprints, and a list of new masks for the ``coaddTempExps``.
1542 
1543  Parameters
1544  ----------
1545  exp : `lsst.afw.image.Exposure`
1546  Exposure to run detection on.
1547  tempExpRefList : `list`
1548  List of data reference to tempExp.
1549 
1550  Returns
1551  -------
1552  result : `lsst.pipe.base.Struct`
1553  Result struct with components:
1554 
1555  - ``clipFootprints``: list of clipped footprints.
1556  - ``clipIndices``: indices for each ``clippedFootprint`` in
1557  ``tempExpRefList``.
1558  - ``clipSpans``: List of dictionaries containing spanSet lists
1559  to clip. Each element contains the new maskplane name
1560  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1561  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1562  compressed into footprints.
1563  """
1564  mask = exp.getMaskedImage().getMask()
1565  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1566  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1567  # Merge positive and negative together footprints together
1568  fpSet.positive.merge(fpSet.negative)
1569  footprints = fpSet.positive
1570  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1571  ignoreMask = self.getBadPixelMask()
1572 
1573  clipFootprints = []
1574  clipIndices = []
1575  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1576 
1577  # for use by detectClipBig
1578  visitDetectionFootprints = []
1579 
1580  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1581  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1582  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1583 
1584  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1585  for i, warpRef in enumerate(tempExpRefList):
1586  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1587  immediate=True).getMaskedImage().getMask()
1588  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1589  afwImage.PARENT, True)
1590  maskVisitDet &= maskDetValue
1591  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1592  visitDetectionFootprints.append(visitFootprints)
1593 
1594  for j, footprint in enumerate(footprints.getFootprints()):
1595  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1596  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1597 
1598  # build a list of clipped spans for each visit
1599  for j, footprint in enumerate(footprints.getFootprints()):
1600  nPixel = footprint.getArea()
1601  overlap = [] # hold the overlap with each visit
1602  indexList = [] # index of visit in global list
1603  for i in range(len(tempExpRefList)):
1604  ignore = ignoreArr[i, j]
1605  overlapDet = overlapDetArr[i, j]
1606  totPixel = nPixel - ignore
1607 
1608  # If we have more bad pixels than detection skip
1609  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1610  continue
1611  overlap.append(overlapDet/float(totPixel))
1612  indexList.append(i)
1613 
1614  overlap = numpy.array(overlap)
1615  if not len(overlap):
1616  continue
1617 
1618  keep = False # Should this footprint be marked as clipped?
1619  keepIndex = [] # Which tempExps does the clipped footprint belong to
1620 
1621  # If footprint only has one overlap use a lower threshold
1622  if len(overlap) == 1:
1623  if overlap[0] > self.config.minClipFootOverlapSingle:
1624  keep = True
1625  keepIndex = [0]
1626  else:
1627  # This is the general case where only visit should be clipped
1628  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1629  if len(clipIndex) == 1:
1630  keep = True
1631  keepIndex = [clipIndex[0]]
1632 
1633  # Test if there are clipped objects that overlap two different visits
1634  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1635  if len(clipIndex) == 2 and len(overlap) > 3:
1636  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1637  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1638  keep = True
1639  keepIndex = clipIndex
1640 
1641  if not keep:
1642  continue
1643 
1644  for index in keepIndex:
1645  globalIndex = indexList[index]
1646  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1647 
1648  clipIndices.append(numpy.array(indexList)[keepIndex])
1649  clipFootprints.append(footprint)
1650 
1651  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1652  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1653 
1654  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1655  maskClipValue, maskDetValue, coaddBBox):
1656  """Return individual warp footprints for large artifacts and append
1657  them to ``clipList`` in place.
1658 
1659  Identify big footprints composed of many sources in the coadd
1660  difference that may have originated in a large diffuse source in the
1661  coadd. We do this by indentifying all clipped footprints that overlap
1662  significantly with each source in all the coaddTempExps.
1663 
1664  Parameters
1665  ----------
1666  clipList : `list`
1667  List of alt mask SpanSets with clipping information. Modified.
1668  clipFootprints : `list`
1669  List of clipped footprints.
1670  clipIndices : `list`
1671  List of which entries in tempExpClipList each footprint belongs to.
1672  maskClipValue
1673  Mask value of clipped pixels.
1674  maskDetValue
1675  Mask value of detected pixels.
1676  coaddBBox : `lsst.geom.Box`
1677  BBox of the coadd and warps.
1678 
1679  Returns
1680  -------
1681  bigFootprintsCoadd : `list`
1682  List of big footprints
1683  """
1684  bigFootprintsCoadd = []
1685  ignoreMask = self.getBadPixelMask()
1686  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1687  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1688  for footprint in visitFootprints.getFootprints():
1689  footprint.spans.setMask(maskVisitDet, maskDetValue)
1690 
1691  # build a mask of clipped footprints that are in this visit
1692  clippedFootprintsVisit = []
1693  for foot, clipIndex in zip(clipFootprints, clipIndices):
1694  if index not in clipIndex:
1695  continue
1696  clippedFootprintsVisit.append(foot)
1697  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1698  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1699 
1700  bigFootprintsVisit = []
1701  for foot in visitFootprints.getFootprints():
1702  if foot.getArea() < self.config.minBigOverlap:
1703  continue
1704  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1705  if nCount > self.config.minBigOverlap:
1706  bigFootprintsVisit.append(foot)
1707  bigFootprintsCoadd.append(foot)
1708 
1709  for footprint in bigFootprintsVisit:
1710  clippedSpans["CLIPPED"].append(footprint.spans)
1711 
1712  return bigFootprintsCoadd
1713 
1714 
1716  psfMatchedWarps = pipeBase.connectionTypes.Input(
1717  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1718  "Only PSF-Matched Warps make sense for image subtraction. "
1719  "Therefore, they must be an additional declared input."),
1720  name="{inputCoaddName}Coadd_psfMatchedWarp",
1721  storageClass="ExposureF",
1722  dimensions=("tract", "patch", "skymap", "visit"),
1723  deferLoad=True,
1724  multiple=True
1725  )
1726  templateCoadd = pipeBase.connectionTypes.Output(
1727  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1728  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1729  name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1730  storageClass="ExposureF",
1731  dimensions=("tract", "patch", "skymap", "band"),
1732  )
1733 
1734  def __init__(self, *, config=None):
1735  super().__init__(config=config)
1736  if not config.assembleStaticSkyModel.doWrite:
1737  self.outputs.remove("templateCoadd")
1738  config.validate()
1739 
1740 
1741 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1742  pipelineConnections=CompareWarpAssembleCoaddConnections):
1743  assembleStaticSkyModel = pexConfig.ConfigurableField(
1744  target=AssembleCoaddTask,
1745  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1746  " naive/first-iteration model of the static sky.",
1747  )
1748  detect = pexConfig.ConfigurableField(
1749  target=SourceDetectionTask,
1750  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1751  )
1752  detectTemplate = pexConfig.ConfigurableField(
1753  target=SourceDetectionTask,
1754  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1755  )
1756  maskStreaks = pexConfig.ConfigurableField(
1757  target=MaskStreaksTask,
1758  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1759  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1760  "streakMaskName"
1761  )
1762  streakMaskName = pexConfig.Field(
1763  dtype=str,
1764  default="STREAK",
1765  doc="Name of mask bit used for streaks"
1766  )
1767  maxNumEpochs = pexConfig.Field(
1768  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1769  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1770  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1771  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1772  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1773  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1774  "than transient and not masked.",
1775  dtype=int,
1776  default=2
1777  )
1778  maxFractionEpochsLow = pexConfig.RangeField(
1779  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1780  "Effective maxNumEpochs = "
1781  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1782  dtype=float,
1783  default=0.4,
1784  min=0., max=1.,
1785  )
1786  maxFractionEpochsHigh = pexConfig.RangeField(
1787  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1788  "Effective maxNumEpochs = "
1789  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1790  dtype=float,
1791  default=0.03,
1792  min=0., max=1.,
1793  )
1794  spatialThreshold = pexConfig.RangeField(
1795  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1796  "temporal criteria. If 0, clip all. If 1, clip none.",
1797  dtype=float,
1798  default=0.5,
1799  min=0., max=1.,
1800  inclusiveMin=True, inclusiveMax=True
1801  )
1802  doScaleWarpVariance = pexConfig.Field(
1803  doc="Rescale Warp variance plane using empirical noise?",
1804  dtype=bool,
1805  default=True,
1806  )
1807  scaleWarpVariance = pexConfig.ConfigurableField(
1808  target=ScaleVarianceTask,
1809  doc="Rescale variance on warps",
1810  )
1811  doPreserveContainedBySource = pexConfig.Field(
1812  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1813  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1814  dtype=bool,
1815  default=True,
1816  )
1817  doPrefilterArtifacts = pexConfig.Field(
1818  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1819  "because they will be excluded anyway. This prevents them from contributing "
1820  "to the outlier epoch count image and potentially being labeled as persistant."
1821  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1822  dtype=bool,
1823  default=True
1824  )
1825  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1826  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1827  dtype=str,
1828  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1829  )
1830  prefilterArtifactsRatio = pexConfig.Field(
1831  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1832  dtype=float,
1833  default=0.05
1834  )
1835  doFilterMorphological = pexConfig.Field(
1836  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1837  "be streaks.",
1838  dtype=bool,
1839  default=False
1840  )
1841 
1842  def setDefaults(self):
1843  AssembleCoaddConfig.setDefaults(self)
1844  self.statisticstatistic = 'MEAN'
1845  self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
1846 
1847  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1848  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1849  if "EDGE" in self.badMaskPlanes:
1850  self.badMaskPlanes.remove('EDGE')
1851  self.removeMaskPlanes.append('EDGE')
1852  self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1853  self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
1854  self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
1855  self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
1856  self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
1857  self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
1858  self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
1859  self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
1860  self.detectdetect.doTempLocalBackground = False
1861  self.detectdetect.reEstimateBackground = False
1862  self.detectdetect.returnOriginalFootprints = False
1863  self.detectdetect.thresholdPolarity = "both"
1864  self.detectdetect.thresholdValue = 5
1865  self.detectdetect.minPixels = 4
1866  self.detectdetect.isotropicGrow = True
1867  self.detectdetect.thresholdType = "pixel_stdev"
1868  self.detectdetect.nSigmaToGrow = 0.4
1869  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1870  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1871  self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
1872  self.detectTemplatedetectTemplate.doTempLocalBackground = False
1873  self.detectTemplatedetectTemplate.reEstimateBackground = False
1874  self.detectTemplatedetectTemplate.returnOriginalFootprints = False
1875 
1876  def validate(self):
1877  super().validate()
1878  if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
1879  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1880  "Please set assembleStaticSkyModel.doNImage=False")
1881 
1882  if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
1883  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1884  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1885  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1886  "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
1887 
1888 
1889 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1890  """Assemble a compareWarp coadded image from a set of warps
1891  by masking artifacts detected by comparing PSF-matched warps.
1892 
1893  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1894  we clip outliers). The problem with doing this is that when computing the
1895  coadd PSF at a given location, individual visit PSFs from visits with
1896  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1897  In this task, we correct for this behavior by creating a new badMaskPlane
1898  'CLIPPED' which marks pixels in the individual warps suspected to contain
1899  an artifact. We populate this plane on the input warps by comparing
1900  PSF-matched warps with a PSF-matched median coadd which serves as a
1901  model of the static sky. Any group of pixels that deviates from the
1902  PSF-matched template coadd by more than config.detect.threshold sigma,
1903  is an artifact candidate. The candidates are then filtered to remove
1904  variable sources and sources that are difficult to subtract such as
1905  bright stars. This filter is configured using the config parameters
1906  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1907  the maximum fraction of epochs that the deviation can appear in and still
1908  be considered an artifact. The spatialThreshold is the maximum fraction of
1909  pixels in the footprint of the deviation that appear in other epochs
1910  (where other epochs is defined by the temporalThreshold). If the deviant
1911  region meets this criteria of having a significant percentage of pixels
1912  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1913  set in the mask. These regions will not contribute to the final coadd.
1914  Furthermore, any routine to determine the coadd PSF can now be cognizant
1915  of clipped regions. Note that the algorithm implemented by this task is
1916  preliminary and works correctly for HSC data. Parameter modifications and
1917  or considerable redesigning of the algorithm is likley required for other
1918  surveys.
1919 
1920  ``CompareWarpAssembleCoaddTask`` sub-classes
1921  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1922  as a subtask to generate the TemplateCoadd (the model of the static sky).
1923 
1924  Notes
1925  -----
1926  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1927  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1928  ``baseDebug`` for more about ``debug.py`` files.
1929 
1930  This task supports the following debug variables:
1931 
1932  - ``saveCountIm``
1933  If True then save the Epoch Count Image as a fits file in the `figPath`
1934  - ``figPath``
1935  Path to save the debug fits images and figures
1936 
1937  For example, put something like:
1938 
1939  .. code-block:: python
1940 
1941  import lsstDebug
1942  def DebugInfo(name):
1943  di = lsstDebug.getInfo(name)
1944  if name == "lsst.pipe.tasks.assembleCoadd":
1945  di.saveCountIm = True
1946  di.figPath = "/desired/path/to/debugging/output/images"
1947  return di
1948  lsstDebug.Info = DebugInfo
1949 
1950  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1951  ``--debug`` flag. Some subtasks may have their own debug variables;
1952  see individual Task documentation.
1953 
1954  Examples
1955  --------
1956  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1957  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1958  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1959  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1960  and filter to be coadded (specified using
1961  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1962  along with a list of coaddTempExps to attempt to coadd (specified using
1963  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1964  Only the warps that cover the specified tract and patch will be coadded.
1965  A list of the available optional arguments can be obtained by calling
1966  ``assembleCoadd.py`` with the ``--help`` command line argument:
1967 
1968  .. code-block:: none
1969 
1970  assembleCoadd.py --help
1971 
1972  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1973  context of multi-band processing, we will generate the HSC-I & -R band
1974  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1975  To begin, assuming that the lsst stack has been already set up, we must
1976  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1977  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1978  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1979  directory. To begin assembling the coadds, we must first
1980 
1981  - processCcd
1982  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1983  - makeSkyMap
1984  create a skymap that covers the area of the sky present in the raw exposures
1985  - makeCoaddTempExp
1986  warp the individual calibrated exposures to the tangent plane of the coadd
1987 
1988  We can perform all of these steps by running
1989 
1990  .. code-block:: none
1991 
1992  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1993 
1994  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1995  warped data, we call ``assembleCoadd.py`` as follows:
1996 
1997  .. code-block:: none
1998 
1999  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2000  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2001  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2002  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2003  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2004  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2005  --selectId visit=903988 ccd=24
2006 
2007  This will process the HSC-I band data. The results are written in
2008  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2009  """
2010  ConfigClass = CompareWarpAssembleCoaddConfig
2011  _DefaultName = "compareWarpAssembleCoadd"
2012 
2013  def __init__(self, *args, **kwargs):
2014  AssembleCoaddTask.__init__(self, *args, **kwargs)
2015  self.makeSubtask("assembleStaticSkyModel")
2016  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2017  self.makeSubtask("detect", schema=detectionSchema)
2018  if self.config.doPreserveContainedBySource:
2019  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2020  if self.config.doScaleWarpVariance:
2021  self.makeSubtask("scaleWarpVariance")
2022  if self.config.doFilterMorphological:
2023  self.makeSubtask("maskStreaks")
2024 
2025  @utils.inheritDoc(AssembleCoaddTask)
2026  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2027  """
2028  Generate a templateCoadd to use as a naive model of static sky to
2029  subtract from PSF-Matched warps.
2030 
2031  Returns
2032  -------
2033  result : `lsst.pipe.base.Struct`
2034  Result struct with components:
2035 
2036  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2037  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2038  """
2039  # Ensure that psfMatchedWarps are used as input warps for template generation
2040  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2041  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2042 
2043  # Because subtasks don't have connections we have to make one.
2044  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2045  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2046  if self.config.assembleStaticSkyModel.doWrite:
2047  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2048  # Remove template coadd from both subtask's and main tasks outputs,
2049  # because it is handled by the subtask as `coaddExposure`
2050  del outputRefs.templateCoadd
2051  del staticSkyModelOutputRefs.templateCoadd
2052 
2053  # A PSF-Matched nImage does not exist as a dataset type
2054  if 'nImage' in staticSkyModelOutputRefs.keys():
2055  del staticSkyModelOutputRefs.nImage
2056 
2057  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2058  staticSkyModelOutputRefs)
2059  if templateCoadd is None:
2060  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2061 
2062  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2063  nImage=templateCoadd.nImage,
2064  warpRefList=templateCoadd.warpRefList,
2065  imageScalerList=templateCoadd.imageScalerList,
2066  weightList=templateCoadd.weightList)
2067 
2068  @utils.inheritDoc(AssembleCoaddTask)
2069  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2070  """
2071  Generate a templateCoadd to use as a naive model of static sky to
2072  subtract from PSF-Matched warps.
2073 
2074  Returns
2075  -------
2076  result : `lsst.pipe.base.Struct`
2077  Result struct with components:
2078 
2079  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2080  - ``nImage``: N Image (``lsst.afw.image.Image``)
2081  """
2082  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2083  if templateCoadd is None:
2084  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2085 
2086  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2087  nImage=templateCoadd.nImage,
2088  warpRefList=templateCoadd.warpRefList,
2089  imageScalerList=templateCoadd.imageScalerList,
2090  weightList=templateCoadd.weightList)
2091 
2092  def _noTemplateMessage(self, warpType):
2093  warpName = (warpType[0].upper() + warpType[1:])
2094  message = """No %(warpName)s warps were found to build the template coadd which is
2095  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2096  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2097  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2098 
2099  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2100  another algorithm like:
2101 
2102  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2103  config.assemble.retarget(SafeClipAssembleCoaddTask)
2104  """ % {"warpName": warpName}
2105  return message
2106 
2107  @utils.inheritDoc(AssembleCoaddTask)
2108  @pipeBase.timeMethod
2109  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2110  supplementaryData, *args, **kwargs):
2111  """Assemble the coadd.
2112 
2113  Find artifacts and apply them to the warps' masks creating a list of
2114  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2115  plane. Then pass these alternative masks to the base class's `run`
2116  method.
2117 
2118  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2119  that must contain a ``templateCoadd`` that serves as the
2120  model of the static sky.
2121  """
2122 
2123  # Check and match the order of the supplementaryData
2124  # (PSF-matched) inputs to the order of the direct inputs,
2125  # so that the artifact mask is applied to the right warp
2126  dataIds = [ref.dataId for ref in tempExpRefList]
2127  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2128 
2129  if dataIds != psfMatchedDataIds:
2130  self.log.info("Reordering and or/padding PSF-matched visit input list")
2131  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2132  psfMatchedDataIds, dataIds)
2133  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2134  psfMatchedDataIds, dataIds)
2135 
2136  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2137  spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2138  supplementaryData.warpRefList,
2139  supplementaryData.imageScalerList)
2140 
2141  badMaskPlanes = self.config.badMaskPlanes[:]
2142  badMaskPlanes.append("CLIPPED")
2143  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2144 
2145  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2146  spanSetMaskList, mask=badPixelMask)
2147 
2148  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2149  # Psf-Matching moves the real edge inwards
2150  self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2151  return result
2152 
2153  def applyAltEdgeMask(self, mask, altMaskList):
2154  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2155 
2156  Parameters
2157  ----------
2158  mask : `lsst.afw.image.Mask`
2159  Original mask.
2160  altMaskList : `list`
2161  List of Dicts containing ``spanSet`` lists.
2162  Each element contains the new mask plane name (e.g. "CLIPPED
2163  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2164  the mask.
2165  """
2166  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2167  for visitMask in altMaskList:
2168  if "EDGE" in visitMask:
2169  for spanSet in visitMask['EDGE']:
2170  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2171 
2172  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2173  """Find artifacts.
2174 
2175  Loop through warps twice. The first loop builds a map with the count
2176  of how many epochs each pixel deviates from the templateCoadd by more
2177  than ``config.chiThreshold`` sigma. The second loop takes each
2178  difference image and filters the artifacts detected in each using
2179  count map to filter out variable sources and sources that are
2180  difficult to subtract cleanly.
2181 
2182  Parameters
2183  ----------
2184  templateCoadd : `lsst.afw.image.Exposure`
2185  Exposure to serve as model of static sky.
2186  tempExpRefList : `list`
2187  List of data references to warps.
2188  imageScalerList : `list`
2189  List of image scalers.
2190 
2191  Returns
2192  -------
2193  altMasks : `list`
2194  List of dicts containing information about CLIPPED
2195  (i.e., artifacts), NO_DATA, and EDGE pixels.
2196  """
2197 
2198  self.log.debug("Generating Count Image, and mask lists.")
2199  coaddBBox = templateCoadd.getBBox()
2200  slateIm = afwImage.ImageU(coaddBBox)
2201  epochCountImage = afwImage.ImageU(coaddBBox)
2202  nImage = afwImage.ImageU(coaddBBox)
2203  spanSetArtifactList = []
2204  spanSetNoDataMaskList = []
2205  spanSetEdgeList = []
2206  spanSetBadMorphoList = []
2207  badPixelMask = self.getBadPixelMask()
2208 
2209  # mask of the warp diffs should = that of only the warp
2210  templateCoadd.mask.clearAllMaskPlanes()
2211 
2212  if self.config.doPreserveContainedBySource:
2213  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2214  else:
2215  templateFootprints = None
2216 
2217  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2218  warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2219  if warpDiffExp is not None:
2220  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2221  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2222  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2223  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2224  fpSet.positive.merge(fpSet.negative)
2225  footprints = fpSet.positive
2226  slateIm.set(0)
2227  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2228 
2229  # Remove artifacts due to defects before they contribute to the epochCountImage
2230  if self.config.doPrefilterArtifacts:
2231  spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2232 
2233  # Clear mask before adding prefiltered spanSets
2234  self.detect.clearMask(warpDiffExp.mask)
2235  for spans in spanSetList:
2236  spans.setImage(slateIm, 1, doClip=True)
2237  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2238  epochCountImage += slateIm
2239 
2240  if self.config.doFilterMorphological:
2241  maskName = self.config.streakMaskName
2242  _ = self.maskStreaks.run(warpDiffExp)
2243  streakMask = warpDiffExp.mask
2244  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2245  streakMask.getPlaneBitMask(maskName)).split()
2246 
2247  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2248  # undergo a second convolution. Pixels with data in the direct warp
2249  # but not in the PSF-matched warp will not have their artifacts detected.
2250  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2251  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2252  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2253  nansMask.setXY0(warpDiffExp.getXY0())
2254  edgeMask = warpDiffExp.mask
2255  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2256  edgeMask.getPlaneBitMask("EDGE")).split()
2257  else:
2258  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2259  # In this case, mask the whole epoch
2260  nansMask = afwImage.MaskX(coaddBBox, 1)
2261  spanSetList = []
2262  spanSetEdgeMask = []
2263  spanSetStreak = []
2264 
2265  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2266 
2267  spanSetNoDataMaskList.append(spanSetNoDataMask)
2268  spanSetArtifactList.append(spanSetList)
2269  spanSetEdgeList.append(spanSetEdgeMask)
2270  if self.config.doFilterMorphological:
2271  spanSetBadMorphoList.append(spanSetStreak)
2272 
2273  if lsstDebug.Info(__name__).saveCountIm:
2274  path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2275  epochCountImage.writeFits(path)
2276 
2277  for i, spanSetList in enumerate(spanSetArtifactList):
2278  if spanSetList:
2279  filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2280  templateFootprints)
2281  spanSetArtifactList[i] = filteredSpanSetList
2282  if self.config.doFilterMorphological:
2283  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2284 
2285  altMasks = []
2286  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2287  altMasks.append({'CLIPPED': artifacts,
2288  'NO_DATA': noData,
2289  'EDGE': edge})
2290  return altMasks
2291 
2292  def prefilterArtifacts(self, spanSetList, exp):
2293  """Remove artifact candidates covered by bad mask plane.
2294 
2295  Any future editing of the candidate list that does not depend on
2296  temporal information should go in this method.
2297 
2298  Parameters
2299  ----------
2300  spanSetList : `list`
2301  List of SpanSets representing artifact candidates.
2302  exp : `lsst.afw.image.Exposure`
2303  Exposure containing mask planes used to prefilter.
2304 
2305  Returns
2306  -------
2307  returnSpanSetList : `list`
2308  List of SpanSets with artifacts.
2309  """
2310  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2311  goodArr = (exp.mask.array & badPixelMask) == 0
2312  returnSpanSetList = []
2313  bbox = exp.getBBox()
2314  x0, y0 = exp.getXY0()
2315  for i, span in enumerate(spanSetList):
2316  y, x = span.clippedTo(bbox).indices()
2317  yIndexLocal = numpy.array(y) - y0
2318  xIndexLocal = numpy.array(x) - x0
2319  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2320  if goodRatio > self.config.prefilterArtifactsRatio:
2321  returnSpanSetList.append(span)
2322  return returnSpanSetList
2323 
2324  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2325  """Filter artifact candidates.
2326 
2327  Parameters
2328  ----------
2329  spanSetList : `list`
2330  List of SpanSets representing artifact candidates.
2331  epochCountImage : `lsst.afw.image.Image`
2332  Image of accumulated number of warpDiff detections.
2333  nImage : `lsst.afw.image.Image`
2334  Image of the accumulated number of total epochs contributing.
2335 
2336  Returns
2337  -------
2338  maskSpanSetList : `list`
2339  List of SpanSets with artifacts.
2340  """
2341 
2342  maskSpanSetList = []
2343  x0, y0 = epochCountImage.getXY0()
2344  for i, span in enumerate(spanSetList):
2345  y, x = span.indices()
2346  yIdxLocal = [y1 - y0 for y1 in y]
2347  xIdxLocal = [x1 - x0 for x1 in x]
2348  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2349  totalN = nImage.array[yIdxLocal, xIdxLocal]
2350 
2351  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2352  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2353  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2354  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2355  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2356  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2357  & (outlierN <= effectiveMaxNumEpochs))
2358  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2359  if percentBelowThreshold > self.config.spatialThreshold:
2360  maskSpanSetList.append(span)
2361 
2362  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2363  # If a candidate is contained by a footprint on the template coadd, do not clip
2364  filteredMaskSpanSetList = []
2365  for span in maskSpanSetList:
2366  doKeep = True
2367  for footprint in footprintsToExclude.positive.getFootprints():
2368  if footprint.spans.contains(span):
2369  doKeep = False
2370  break
2371  if doKeep:
2372  filteredMaskSpanSetList.append(span)
2373  maskSpanSetList = filteredMaskSpanSetList
2374 
2375  return maskSpanSetList
2376 
2377  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2378  """Fetch a warp from the butler and return a warpDiff.
2379 
2380  Parameters
2381  ----------
2382  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2383  Butler dataRef for the warp.
2384  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2385  An image scaler object.
2386  templateCoadd : `lsst.afw.image.Exposure`
2387  Exposure to be substracted from the scaled warp.
2388 
2389  Returns
2390  -------
2391  warp : `lsst.afw.image.Exposure`
2392  Exposure of the image difference between the warp and template.
2393  """
2394 
2395  # If the PSF-Matched warp did not exist for this direct warp
2396  # None is holding its place to maintain order in Gen 3
2397  if warpRef is None:
2398  return None
2399  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2400  warpName = self.getTempExpDatasetName('psfMatched')
2401  if not isinstance(warpRef, DeferredDatasetHandle):
2402  if not warpRef.datasetExists(warpName):
2403  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2404  return None
2405  warp = warpRef.get(datasetType=warpName, immediate=True)
2406  # direct image scaler OK for PSF-matched Warp
2407  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2408  mi = warp.getMaskedImage()
2409  if self.config.doScaleWarpVariance:
2410  try:
2411  self.scaleWarpVariance.run(mi)
2412  except Exception as exc:
2413  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2414  mi -= templateCoadd.getMaskedImage()
2415  return warp
2416 
2417  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2418  """Return a path to which to write debugging output.
2419 
2420  Creates a hyphen-delimited string of dataId values for simple filenames.
2421 
2422  Parameters
2423  ----------
2424  prefix : `str`
2425  Prefix for filename.
2426  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2427  Butler dataRef to make the path from.
2428  coaddLevel : `bool`, optional.
2429  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2430  'filter', but no 'visit').
2431 
2432  Returns
2433  -------
2434  result : `str`
2435  Path for debugging output.
2436  """
2437  if coaddLevel:
2438  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2439  else:
2440  keys = warpRef.dataId.keys()
2441  keyList = sorted(keys, reverse=True)
2442  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2443  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2444  return os.path.join(directory, filename)
2445 
2446 
2447 def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2448  """Match the order of one list to another, padding if necessary
2449 
2450  Parameters
2451  ----------
2452  inputList : list
2453  List to be reordered and padded. Elements can be any type.
2454  inputKeys : iterable
2455  Iterable of values to be compared with outputKeys.
2456  Length must match `inputList`
2457  outputKeys : iterable
2458  Iterable of values to be compared with inputKeys.
2459  padWith :
2460  Any value to be inserted where inputKey not in outputKeys
2461 
2462  Returns
2463  -------
2464  list
2465  Copy of inputList reordered per outputKeys and padded with `padWith`
2466  so that the length matches length of outputKeys.
2467  """
2468  outputList = []
2469  for d in outputKeys:
2470  if d in inputKeys:
2471  outputList.append(inputList[inputKeys.index(d)])
2472  else:
2473  outputList.append(padWith)
2474  return outputList
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:131
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:336
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60