lsst.pipe.tasks  21.0.0-31-g3fcf0262+ee4727a31a
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from lsst.meas.algorithms import SourceDetectionTask
48 from lsst.daf.butler import DeferredDatasetHandle
49 
50 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
51  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
52  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
53 
54 
55 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
56  dimensions=("tract", "patch", "band", "skymap"),
57  defaultTemplates={"inputCoaddName": "deep",
58  "outputCoaddName": "deep",
59  "warpType": "direct",
60  "warpTypeSuffix": "",
61  "fakesType": ""}):
62  inputWarps = pipeBase.connectionTypes.Input(
63  doc=("Input list of warps to be assemebled i.e. stacked."
64  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
65  name="{inputCoaddName}Coadd_{warpType}Warp",
66  storageClass="ExposureF",
67  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
68  deferLoad=True,
69  multiple=True
70  )
71  skyMap = pipeBase.connectionTypes.Input(
72  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
73  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
74  storageClass="SkyMap",
75  dimensions=("skymap", ),
76  )
77  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
78  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
79  " BRIGHT_OBJECT."),
80  name="brightObjectMask",
81  storageClass="ObjectMaskCatalog",
82  dimensions=("tract", "patch", "skymap", "band"),
83  )
84  coaddExposure = pipeBase.connectionTypes.Output(
85  doc="Output coadded exposure, produced by stacking input warps",
86  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
87  storageClass="ExposureF",
88  dimensions=("tract", "patch", "skymap", "band"),
89  )
90  nImage = pipeBase.connectionTypes.Output(
91  doc="Output image of number of input images per pixel",
92  name="{outputCoaddName}Coadd_nImage",
93  storageClass="ImageU",
94  dimensions=("tract", "patch", "skymap", "band"),
95  )
96 
97  def __init__(self, *, config=None):
98  super().__init__(config=config)
99 
100  # Override the connection's name template with config to replicate Gen2 behavior
101  # This duplicates some of the logic in the base class, due to wanting Gen2 and
102  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
103  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
104  templateValues['warpType'] = config.warpType
105  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
106  if config.hasFakes:
107  templateValues['fakesType'] = "_fakes"
108  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
109  for name in self.allConnections}
110  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
111  # End code to remove after deprecation
112 
113  if not config.doMaskBrightObjects:
114  self.prerequisiteInputs.remove("brightObjectMask")
115 
116  if not config.doNImage:
117  self.outputs.remove("nImage")
118 
119 
120 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
121  pipelineConnections=AssembleCoaddConnections):
122  """Configuration parameters for the `AssembleCoaddTask`.
123 
124  Notes
125  -----
126  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
127  only set the bitplane config.brightObjectMaskName. To make this useful you
128  *must* also configure the flags.pixel algorithm, for example by adding
129 
130  .. code-block:: none
131 
132  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
133  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
134 
135  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
136  """
137  warpType = pexConfig.Field(
138  doc="Warp name: one of 'direct' or 'psfMatched'",
139  dtype=str,
140  default="direct",
141  )
142  subregionSize = pexConfig.ListField(
143  dtype=int,
144  doc="Width, height of stack subregion size; "
145  "make small enough that a full stack of images will fit into memory at once.",
146  length=2,
147  default=(2000, 2000),
148  )
149  statistic = pexConfig.Field(
150  dtype=str,
151  doc="Main stacking statistic for aggregating over the epochs.",
152  default="MEANCLIP",
153  )
154  doSigmaClip = pexConfig.Field(
155  dtype=bool,
156  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
157  default=False,
158  )
159  sigmaClip = pexConfig.Field(
160  dtype=float,
161  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
162  default=3.0,
163  )
164  clipIter = pexConfig.Field(
165  dtype=int,
166  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
167  default=2,
168  )
169  calcErrorFromInputVariance = pexConfig.Field(
170  dtype=bool,
171  doc="Calculate coadd variance from input variance by stacking statistic."
172  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
173  default=True,
174  )
175  scaleZeroPoint = pexConfig.ConfigurableField(
176  target=ScaleZeroPointTask,
177  doc="Task to adjust the photometric zero point of the coadd temp exposures",
178  )
179  doInterp = pexConfig.Field(
180  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
181  dtype=bool,
182  default=True,
183  )
184  interpImage = pexConfig.ConfigurableField(
185  target=InterpImageTask,
186  doc="Task to interpolate (and extrapolate) over NaN pixels",
187  )
188  doWrite = pexConfig.Field(
189  doc="Persist coadd?",
190  dtype=bool,
191  default=True,
192  )
193  doNImage = pexConfig.Field(
194  doc="Create image of number of contributing exposures for each pixel",
195  dtype=bool,
196  default=False,
197  )
198  doUsePsfMatchedPolygons = pexConfig.Field(
199  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
200  dtype=bool,
201  default=False,
202  )
203  maskPropagationThresholds = pexConfig.DictField(
204  keytype=str,
205  itemtype=float,
206  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
207  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
208  "would have contributed exceeds this value."),
209  default={"SAT": 0.1},
210  )
211  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
212  doc="Mask planes to remove before coadding")
213  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
214  doc="Set mask and flag bits for bright objects?")
215  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
216  doc="Name of mask bit used for bright objects")
217  coaddPsf = pexConfig.ConfigField(
218  doc="Configuration for CoaddPsf",
219  dtype=measAlg.CoaddPsfConfig,
220  )
221  doAttachTransmissionCurve = pexConfig.Field(
222  dtype=bool, default=False, optional=False,
223  doc=("Attach a piecewise TransmissionCurve for the coadd? "
224  "(requires all input Exposures to have TransmissionCurves).")
225  )
226  hasFakes = pexConfig.Field(
227  dtype=bool,
228  default=False,
229  doc="Should be set to True if fake sources have been inserted into the input data."
230  )
231 
232  def setDefaults(self):
233  super().setDefaults()
234  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
235 
236  def validate(self):
237  super().validate()
238  if self.doPsfMatch:
239  # Backwards compatibility.
240  # Configs do not have loggers
241  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
242  self.warpType = 'psfMatched'
243  if self.doSigmaClip and self.statistic != "MEANCLIP":
244  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
245  self.statistic = "MEANCLIP"
246  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
247  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
248  "compute and set a non-zero coadd variance estimate." % (self.statistic))
249 
250  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
251  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
252  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
253  if str(k) not in unstackableStats]
254  raise ValueError("statistic %s is not allowed. Please choose one of %s."
255  % (self.statistic, stackableStats))
256 
257 
258 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
259  """Assemble a coadded image from a set of warps (coadded temporary exposures).
260 
261  We want to assemble a coadded image from a set of Warps (also called
262  coadded temporary exposures or ``coaddTempExps``).
263  Each input Warp covers a patch on the sky and corresponds to a single
264  run/visit/exposure of the covered patch. We provide the task with a list
265  of Warps (``selectDataList``) from which it selects Warps that cover the
266  specified patch (pointed at by ``dataRef``).
267  Each Warp that goes into a coadd will typically have an independent
268  photometric zero-point. Therefore, we must scale each Warp to set it to
269  a common photometric zeropoint. WarpType may be one of 'direct' or
270  'psfMatched', and the boolean configs `config.makeDirect` and
271  `config.makePsfMatched` set which of the warp types will be coadded.
272  The coadd is computed as a mean with optional outlier rejection.
273  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
274  Finally, Warps can have bad 'NaN' pixels which received no input from the
275  source calExps. We interpolate over these bad (NaN) pixels.
276 
277  `AssembleCoaddTask` uses several sub-tasks. These are
278 
279  - `ScaleZeroPointTask`
280  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
281  - `InterpImageTask`
282  - interpolate across bad pixels (NaN) in the final coadd
283 
284  You can retarget these subtasks if you wish.
285 
286  Notes
287  -----
288  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
289  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
290  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
291  no debug variables of its own. Some of the subtasks may support debug
292  variables. See the documentation for the subtasks for further information.
293 
294  Examples
295  --------
296  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
297  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
298  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
299  inputs: a data reference to the tract patch and filter to be coadded, and
300  a list of Warps to attempt to coadd. These are specified using ``--id`` and
301  ``--selectId``, respectively:
302 
303  .. code-block:: none
304 
305  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
306  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
307 
308  Only the Warps that cover the specified tract and patch will be coadded.
309  A list of the available optional arguments can be obtained by calling
310  ``assembleCoadd.py`` with the ``--help`` command line argument:
311 
312  .. code-block:: none
313 
314  assembleCoadd.py --help
315 
316  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
317  multi-band processing, we will generate the HSC-I & -R band coadds from
318  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
319  assuming that the lsst stack has been already set up, we must set up the
320  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
321  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
322  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
323  coadds, we must first
324 
325  - processCcd
326  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
327  - makeSkyMap
328  - create a skymap that covers the area of the sky present in the raw exposures
329  - makeCoaddTempExp
330  - warp the individual calibrated exposures to the tangent plane of the coadd
331 
332  We can perform all of these steps by running
333 
334  .. code-block:: none
335 
336  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
337 
338  This will produce warped exposures for each visit. To coadd the warped
339  data, we call assembleCoadd.py as follows:
340 
341  .. code-block:: none
342 
343  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
344  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
345  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
346  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
347  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
348  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
349  --selectId visit=903988 ccd=24
350 
351  that will process the HSC-I band data. The results are written in
352  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
353 
354  You may also choose to run:
355 
356  .. code-block:: none
357 
358  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
359  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
360  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
361  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
362  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
363  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
364  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
365  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
366 
367  to generate the coadd for the HSC-R band if you are interested in
368  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
369  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
370  rather than `AssembleCoaddTask` to make the coadd.
371  """
372  ConfigClass = AssembleCoaddConfig
373  _DefaultName = "assembleCoadd"
374 
375  def __init__(self, *args, **kwargs):
376  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
377  if args:
378  argNames = ["config", "name", "parentTask", "log"]
379  kwargs.update({k: v for k, v in zip(argNames, args)})
380  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
381  "PipelineTask will not take positional args" % argNames, FutureWarning)
382 
383  super().__init__(**kwargs)
384  self.makeSubtask("interpImage")
385  self.makeSubtask("scaleZeroPoint")
386 
387  if self.config.doMaskBrightObjects:
388  mask = afwImage.Mask()
389  try:
390  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
391  except pexExceptions.LsstCppException:
392  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
393  mask.getMaskPlaneDict().keys())
394  del mask
395 
396  self.warpType = self.config.warpType
397 
398  @utils.inheritDoc(pipeBase.PipelineTask)
399  def runQuantum(self, butlerQC, inputRefs, outputRefs):
400  # Docstring to be formatted with info from PipelineTask.runQuantum
401  """
402  Notes
403  -----
404  Assemble a coadd from a set of Warps.
405 
406  PipelineTask (Gen3) entry point to Coadd a set of Warps.
407  Analogous to `runDataRef`, it prepares all the data products to be
408  passed to `run`, and processes the results before returning a struct
409  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
410  Therefore, its inputs are accessed subregion by subregion
411  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
412  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
413  correspond to an update in `runDataRef` while both entry points
414  are used.
415  """
416  inputData = butlerQC.get(inputRefs)
417 
418  # Construct skyInfo expected by run
419  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
420  skyMap = inputData["skyMap"]
421  outputDataId = butlerQC.quantum.dataId
422 
423  inputData['skyInfo'] = makeSkyInfo(skyMap,
424  tractId=outputDataId['tract'],
425  patchId=outputDataId['patch'])
426 
427  # Construct list of input Deferred Datasets
428  # These quack a bit like like Gen2 DataRefs
429  warpRefList = inputData['inputWarps']
430  # Perform same middle steps as `runDataRef` does
431  inputs = self.prepareInputs(warpRefList)
432  self.log.info("Found %d %s", len(inputs.tempExpRefList),
433  self.getTempExpDatasetName(self.warpType))
434  if len(inputs.tempExpRefList) == 0:
435  self.log.warn("No coadd temporary exposures found")
436  return
437 
438  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
439  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
440  inputs.weightList, supplementaryData=supplementaryData)
441 
442  inputData.setdefault('brightObjectMask', None)
443  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
444 
445  if self.config.doWrite:
446  butlerQC.put(retStruct, outputRefs)
447  return retStruct
448 
449  @pipeBase.timeMethod
450  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
451  """Assemble a coadd from a set of Warps.
452 
453  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
454  Compute weights to be applied to each Warp and
455  find scalings to match the photometric zeropoint to a reference Warp.
456  Assemble the Warps using `run`. Interpolate over NaNs and
457  optionally write the coadd to disk. Return the coadded exposure.
458 
459  Parameters
460  ----------
461  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
462  Data reference defining the patch for coaddition and the
463  reference Warp (if ``config.autoReference=False``).
464  Used to access the following data products:
465  - ``self.config.coaddName + "Coadd_skyMap"``
466  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
467  - ``self.config.coaddName + "Coadd"``
468  selectDataList : `list`
469  List of data references to Calexps. Data to be coadded will be
470  selected from this list based on overlap with the patch defined
471  by dataRef, grouped by visit, and converted to a list of data
472  references to warps.
473  warpRefList : `list`
474  List of data references to Warps to be coadded.
475  Note: `warpRefList` is just the new name for `tempExpRefList`.
476 
477  Returns
478  -------
479  retStruct : `lsst.pipe.base.Struct`
480  Result struct with components:
481 
482  - ``coaddExposure``: coadded exposure (``Exposure``).
483  - ``nImage``: exposure count image (``Image``).
484  """
485  if selectDataList and warpRefList:
486  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
487  "and which to use is ambiguous. Please pass only one.")
488 
489  skyInfo = self.getSkyInfo(dataRef)
490  if warpRefList is None:
491  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
492  if len(calExpRefList) == 0:
493  self.log.warn("No exposures to coadd")
494  return
495  self.log.info("Coadding %d exposures", len(calExpRefList))
496 
497  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
498 
499  inputData = self.prepareInputs(warpRefList)
500  self.log.info("Found %d %s", len(inputData.tempExpRefList),
501  self.getTempExpDatasetName(self.warpType))
502  if len(inputData.tempExpRefList) == 0:
503  self.log.warn("No coadd temporary exposures found")
504  return
505 
506  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
507 
508  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
509  inputData.weightList, supplementaryData=supplementaryData)
510 
511  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
512  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
513 
514  if self.config.doWrite:
515  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
516  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
517  else:
518  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
519  self.log.info("Persisting %s" % coaddDatasetName)
520  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
521  if self.config.doNImage and retStruct.nImage is not None:
522  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
523 
524  return retStruct
525 
526  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
527  """Interpolate over missing data and mask bright stars.
528 
529  Parameters
530  ----------
531  coaddExposure : `lsst.afw.image.Exposure`
532  The coadded exposure to process.
533  dataRef : `lsst.daf.persistence.ButlerDataRef`
534  Butler data reference for supplementary data.
535  """
536  if self.config.doInterp:
537  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
538  # The variance must be positive; work around for DM-3201.
539  varArray = coaddExposure.variance.array
540  with numpy.errstate(invalid="ignore"):
541  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
542 
543  if self.config.doMaskBrightObjects:
544  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
545 
546  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
547  """Make additional inputs to run() specific to subclasses (Gen2)
548 
549  Duplicates interface of `runDataRef` method
550  Available to be implemented by subclasses only if they need the
551  coadd dataRef for performing preliminary processing before
552  assembling the coadd.
553 
554  Parameters
555  ----------
556  dataRef : `lsst.daf.persistence.ButlerDataRef`
557  Butler data reference for supplementary data.
558  selectDataList : `list` (optional)
559  Optional List of data references to Calexps.
560  warpRefList : `list` (optional)
561  Optional List of data references to Warps.
562  """
563  return pipeBase.Struct()
564 
565  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
566  """Make additional inputs to run() specific to subclasses (Gen3)
567 
568  Duplicates interface of `runQuantum` method.
569  Available to be implemented by subclasses only if they need the
570  coadd dataRef for performing preliminary processing before
571  assembling the coadd.
572 
573  Parameters
574  ----------
575  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
576  Gen3 Butler object for fetching additional data products before
577  running the Task specialized for quantum being processed
578  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
579  Attributes are the names of the connections describing input dataset types.
580  Values are DatasetRefs that task consumes for corresponding dataset type.
581  DataIds are guaranteed to match data objects in ``inputData``.
582  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
583  Attributes are the names of the connections describing output dataset types.
584  Values are DatasetRefs that task is to produce
585  for corresponding dataset type.
586  """
587  return pipeBase.Struct()
588 
589  def getTempExpRefList(self, patchRef, calExpRefList):
590  """Generate list data references corresponding to warped exposures
591  that lie within the patch to be coadded.
592 
593  Parameters
594  ----------
595  patchRef : `dataRef`
596  Data reference for patch.
597  calExpRefList : `list`
598  List of data references for input calexps.
599 
600  Returns
601  -------
602  tempExpRefList : `list`
603  List of Warp/CoaddTempExp data references.
604  """
605  butler = patchRef.getButler()
606  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
607  self.getTempExpDatasetName(self.warpType))
608  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
609  g, groupData.keys) for
610  g in groupData.groups.keys()]
611  return tempExpRefList
612 
613  def prepareInputs(self, refList):
614  """Prepare the input warps for coaddition by measuring the weight for
615  each warp and the scaling for the photometric zero point.
616 
617  Each Warp has its own photometric zeropoint and background variance.
618  Before coadding these Warps together, compute a scale factor to
619  normalize the photometric zeropoint and compute the weight for each Warp.
620 
621  Parameters
622  ----------
623  refList : `list`
624  List of data references to tempExp
625 
626  Returns
627  -------
628  result : `lsst.pipe.base.Struct`
629  Result struct with components:
630 
631  - ``tempExprefList``: `list` of data references to tempExp.
632  - ``weightList``: `list` of weightings.
633  - ``imageScalerList``: `list` of image scalers.
634  """
635  statsCtrl = afwMath.StatisticsControl()
636  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
637  statsCtrl.setNumIter(self.config.clipIter)
638  statsCtrl.setAndMask(self.getBadPixelMask())
639  statsCtrl.setNanSafe(True)
640  # compute tempExpRefList: a list of tempExpRef that actually exist
641  # and weightList: a list of the weight of the associated coadd tempExp
642  # and imageScalerList: a list of scale factors for the associated coadd tempExp
643  tempExpRefList = []
644  weightList = []
645  imageScalerList = []
646  tempExpName = self.getTempExpDatasetName(self.warpType)
647  for tempExpRef in refList:
648  # Gen3's DeferredDatasetHandles are guaranteed to exist and
649  # therefore have no datasetExists() method
650  if not isinstance(tempExpRef, DeferredDatasetHandle):
651  if not tempExpRef.datasetExists(tempExpName):
652  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
653  continue
654 
655  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
656  # Ignore any input warp that is empty of data
657  if numpy.isnan(tempExp.image.array).all():
658  continue
659  maskedImage = tempExp.getMaskedImage()
660  imageScaler = self.scaleZeroPoint.computeImageScaler(
661  exposure=tempExp,
662  dataRef=tempExpRef,
663  )
664  try:
665  imageScaler.scaleMaskedImage(maskedImage)
666  except Exception as e:
667  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
668  continue
669  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
670  afwMath.MEANCLIP, statsCtrl)
671  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
672  weight = 1.0 / float(meanVar)
673  if not numpy.isfinite(weight):
674  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
675  continue
676  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
677 
678  del maskedImage
679  del tempExp
680 
681  tempExpRefList.append(tempExpRef)
682  weightList.append(weight)
683  imageScalerList.append(imageScaler)
684 
685  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
686  imageScalerList=imageScalerList)
687 
688  def prepareStats(self, mask=None):
689  """Prepare the statistics for coadding images.
690 
691  Parameters
692  ----------
693  mask : `int`, optional
694  Bit mask value to exclude from coaddition.
695 
696  Returns
697  -------
698  stats : `lsst.pipe.base.Struct`
699  Statistics structure with the following fields:
700 
701  - ``statsCtrl``: Statistics control object for coadd
702  (`lsst.afw.math.StatisticsControl`)
703  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
704  """
705  if mask is None:
706  mask = self.getBadPixelMask()
707  statsCtrl = afwMath.StatisticsControl()
708  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
709  statsCtrl.setNumIter(self.config.clipIter)
710  statsCtrl.setAndMask(mask)
711  statsCtrl.setNanSafe(True)
712  statsCtrl.setWeighted(True)
713  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
714  for plane, threshold in self.config.maskPropagationThresholds.items():
715  bit = afwImage.Mask.getMaskPlane(plane)
716  statsCtrl.setMaskPropagationThreshold(bit, threshold)
717  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
718  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
719 
720  @pipeBase.timeMethod
721  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
722  altMaskList=None, mask=None, supplementaryData=None):
723  """Assemble a coadd from input warps
724 
725  Assemble the coadd using the provided list of coaddTempExps. Since
726  the full coadd covers a patch (a large area), the assembly is
727  performed over small areas on the image at a time in order to
728  conserve memory usage. Iterate over subregions within the outer
729  bbox of the patch using `assembleSubregion` to stack the corresponding
730  subregions from the coaddTempExps with the statistic specified.
731  Set the edge bits the coadd mask based on the weight map.
732 
733  Parameters
734  ----------
735  skyInfo : `lsst.pipe.base.Struct`
736  Struct with geometric information about the patch.
737  tempExpRefList : `list`
738  List of data references to Warps (previously called CoaddTempExps).
739  imageScalerList : `list`
740  List of image scalers.
741  weightList : `list`
742  List of weights
743  altMaskList : `list`, optional
744  List of alternate masks to use rather than those stored with
745  tempExp.
746  mask : `int`, optional
747  Bit mask value to exclude from coaddition.
748  supplementaryData : lsst.pipe.base.Struct, optional
749  Struct with additional data products needed to assemble coadd.
750  Only used by subclasses that implement `makeSupplementaryData`
751  and override `run`.
752 
753  Returns
754  -------
755  result : `lsst.pipe.base.Struct`
756  Result struct with components:
757 
758  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
759  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
760  - ``warpRefList``: input list of refs to the warps (
761  ``lsst.daf.butler.DeferredDatasetHandle`` or
762  ``lsst.daf.persistence.ButlerDataRef``)
763  (unmodified)
764  - ``imageScalerList``: input list of image scalers (unmodified)
765  - ``weightList``: input list of weights (unmodified)
766  """
767  tempExpName = self.getTempExpDatasetName(self.warpType)
768  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
769  stats = self.prepareStats(mask=mask)
770 
771  if altMaskList is None:
772  altMaskList = [None]*len(tempExpRefList)
773 
774  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
775  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
776  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
777  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
778  coaddMaskedImage = coaddExposure.getMaskedImage()
779  subregionSizeArr = self.config.subregionSize
780  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
781  # if nImage is requested, create a zero one which can be passed to assembleSubregion
782  if self.config.doNImage:
783  nImage = afwImage.ImageU(skyInfo.bbox)
784  else:
785  nImage = None
786  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
787  try:
788  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
789  weightList, altMaskList, stats.flags, stats.ctrl,
790  nImage=nImage)
791  except Exception as e:
792  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
793 
794  self.setInexactPsf(coaddMaskedImage.getMask())
795  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
796  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
797  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
798  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
799  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
800  weightList=weightList)
801 
802  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
803  """Set the metadata for the coadd.
804 
805  This basic implementation sets the filter from the first input.
806 
807  Parameters
808  ----------
809  coaddExposure : `lsst.afw.image.Exposure`
810  The target exposure for the coadd.
811  tempExpRefList : `list`
812  List of data references to tempExp.
813  weightList : `list`
814  List of weights.
815  """
816  assert len(tempExpRefList) == len(weightList), "Length mismatch"
817  tempExpName = self.getTempExpDatasetName(self.warpType)
818  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
819  # (and we need more than just the PropertySet that contains the header), which is not possible
820  # with the current butler (see #2777).
821  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
822 
823  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
824  # Gen 3 API
825  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
826  else:
827  # Gen 2 API. Delete this when Gen 2 retired
828  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
829  for tempExpRef in tempExpRefList]
830  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
831 
832  # Set the coadd FilterLabel to the band of the first input exposure:
833  # Coadds are calibrated, so the physical label is now meaningless.
834  coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
835  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
836  coaddInputs.ccds.reserve(numCcds)
837  coaddInputs.visits.reserve(len(tempExpList))
838 
839  for tempExp, weight in zip(tempExpList, weightList):
840  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
841 
842  if self.config.doUsePsfMatchedPolygons:
843  self.shrinkValidPolygons(coaddInputs)
844 
845  coaddInputs.visits.sort()
846  if self.warpType == "psfMatched":
847  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
848  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
849  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
850  # having the maximum width (sufficient because square)
851  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
852  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
853  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
854  else:
855  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
856  self.config.coaddPsf.makeControl())
857  coaddExposure.setPsf(psf)
858  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
859  coaddExposure.getWcs())
860  coaddExposure.getInfo().setApCorrMap(apCorrMap)
861  if self.config.doAttachTransmissionCurve:
862  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
863  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
864 
865  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
866  altMaskList, statsFlags, statsCtrl, nImage=None):
867  """Assemble the coadd for a sub-region.
868 
869  For each coaddTempExp, check for (and swap in) an alternative mask
870  if one is passed. Remove mask planes listed in
871  `config.removeMaskPlanes`. Finally, stack the actual exposures using
872  `lsst.afw.math.statisticsStack` with the statistic specified by
873  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
874  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
875  an N-sigma clipped mean where N and iterations are specified by
876  statsCtrl. Assign the stacked subregion back to the coadd.
877 
878  Parameters
879  ----------
880  coaddExposure : `lsst.afw.image.Exposure`
881  The target exposure for the coadd.
882  bbox : `lsst.geom.Box`
883  Sub-region to coadd.
884  tempExpRefList : `list`
885  List of data reference to tempExp.
886  imageScalerList : `list`
887  List of image scalers.
888  weightList : `list`
889  List of weights.
890  altMaskList : `list`
891  List of alternate masks to use rather than those stored with
892  tempExp, or None. Each element is dict with keys = mask plane
893  name to which to add the spans.
894  statsFlags : `lsst.afw.math.Property`
895  Property object for statistic for coadd.
896  statsCtrl : `lsst.afw.math.StatisticsControl`
897  Statistics control object for coadd.
898  nImage : `lsst.afw.image.ImageU`, optional
899  Keeps track of exposure count for each pixel.
900  """
901  self.log.debug("Computing coadd over %s", bbox)
902  tempExpName = self.getTempExpDatasetName(self.warpType)
903  coaddExposure.mask.addMaskPlane("REJECTED")
904  coaddExposure.mask.addMaskPlane("CLIPPED")
905  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
906  maskMap = self.setRejectedMaskMapping(statsCtrl)
907  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
908  maskedImageList = []
909  if nImage is not None:
910  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
911  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
912 
913  if isinstance(tempExpRef, DeferredDatasetHandle):
914  # Gen 3 API
915  exposure = tempExpRef.get(parameters={'bbox': bbox})
916  else:
917  # Gen 2 API. Delete this when Gen 2 retired
918  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
919 
920  maskedImage = exposure.getMaskedImage()
921  mask = maskedImage.getMask()
922  if altMask is not None:
923  self.applyAltMaskPlanes(mask, altMask)
924  imageScaler.scaleMaskedImage(maskedImage)
925 
926  # Add 1 for each pixel which is not excluded by the exclude mask.
927  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
928  if nImage is not None:
929  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
930  if self.config.removeMaskPlanes:
931  self.removeMaskPlanes(maskedImage)
932  maskedImageList.append(maskedImage)
933 
934  with self.timer("stack"):
935  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
936  clipped, # also set output to CLIPPED if sigma-clipped
937  maskMap)
938  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
939  if nImage is not None:
940  nImage.assign(subNImage, bbox)
941 
942  def removeMaskPlanes(self, maskedImage):
943  """Unset the mask of an image for mask planes specified in the config.
944 
945  Parameters
946  ----------
947  maskedImage : `lsst.afw.image.MaskedImage`
948  The masked image to be modified.
949  """
950  mask = maskedImage.getMask()
951  for maskPlane in self.config.removeMaskPlanes:
952  try:
953  mask &= ~mask.getPlaneBitMask(maskPlane)
954  except pexExceptions.InvalidParameterError:
955  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
956  maskPlane)
957 
958  @staticmethod
959  def setRejectedMaskMapping(statsCtrl):
960  """Map certain mask planes of the warps to new planes for the coadd.
961 
962  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
963  or CLIPPED, set it to REJECTED on the coadd.
964  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
965  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
966 
967  Parameters
968  ----------
969  statsCtrl : `lsst.afw.math.StatisticsControl`
970  Statistics control object for coadd
971 
972  Returns
973  -------
974  maskMap : `list` of `tuple` of `int`
975  A list of mappings of mask planes of the warped exposures to
976  mask planes of the coadd.
977  """
978  edge = afwImage.Mask.getPlaneBitMask("EDGE")
979  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
980  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
981  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
982  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
983  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
984  (clipped, clipped)]
985  return maskMap
986 
987  def applyAltMaskPlanes(self, mask, altMaskSpans):
988  """Apply in place alt mask formatted as SpanSets to a mask.
989 
990  Parameters
991  ----------
992  mask : `lsst.afw.image.Mask`
993  Original mask.
994  altMaskSpans : `dict`
995  SpanSet lists to apply. Each element contains the new mask
996  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
997  and list of SpanSets to apply to the mask.
998 
999  Returns
1000  -------
1001  mask : `lsst.afw.image.Mask`
1002  Updated mask.
1003  """
1004  if self.config.doUsePsfMatchedPolygons:
1005  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1006  # Clear away any other masks outside the validPolygons. These pixels are no longer
1007  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1008  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1009  # This mask-clearing step must only occur *before* applying the new masks below
1010  for spanSet in altMaskSpans['NO_DATA']:
1011  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1012 
1013  for plane, spanSetList in altMaskSpans.items():
1014  maskClipValue = mask.addMaskPlane(plane)
1015  for spanSet in spanSetList:
1016  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1017  return mask
1018 
1019  def shrinkValidPolygons(self, coaddInputs):
1020  """Shrink coaddInputs' ccds' ValidPolygons in place.
1021 
1022  Either modify each ccd's validPolygon in place, or if CoaddInputs
1023  does not have a validPolygon, create one from its bbox.
1024 
1025  Parameters
1026  ----------
1027  coaddInputs : `lsst.afw.image.coaddInputs`
1028  Original mask.
1029 
1030  """
1031  for ccd in coaddInputs.ccds:
1032  polyOrig = ccd.getValidPolygon()
1033  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1034  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1035  if polyOrig:
1036  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1037  else:
1038  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1039  ccd.setValidPolygon(validPolygon)
1040 
1041  def readBrightObjectMasks(self, dataRef):
1042  """Retrieve the bright object masks.
1043 
1044  Returns None on failure.
1045 
1046  Parameters
1047  ----------
1048  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1049  A Butler dataRef.
1050 
1051  Returns
1052  -------
1053  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1054  Bright object mask from the Butler object, or None if it cannot
1055  be retrieved.
1056  """
1057  try:
1058  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1059  except Exception as e:
1060  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1061  return None
1062 
1063  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1064  """Set the bright object masks.
1065 
1066  Parameters
1067  ----------
1068  exposure : `lsst.afw.image.Exposure`
1069  Exposure under consideration.
1070  dataId : `lsst.daf.persistence.dataId`
1071  Data identifier dict for patch.
1072  brightObjectMasks : `lsst.afw.table`
1073  Table of bright objects to mask.
1074  """
1075 
1076  if brightObjectMasks is None:
1077  self.log.warn("Unable to apply bright object mask: none supplied")
1078  return
1079  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1080  mask = exposure.getMaskedImage().getMask()
1081  wcs = exposure.getWcs()
1082  plateScale = wcs.getPixelScale().asArcseconds()
1083 
1084  for rec in brightObjectMasks:
1085  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1086  if rec["type"] == "box":
1087  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1088  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1089  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1090 
1091  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1092  bbox = geom.Box2I(center - halfSize, center + halfSize)
1093 
1094  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1095  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1096  spans = afwGeom.SpanSet(bbox)
1097  elif rec["type"] == "circle":
1098  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1099  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1100  else:
1101  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1102  continue
1103  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1104 
1105  def setInexactPsf(self, mask):
1106  """Set INEXACT_PSF mask plane.
1107 
1108  If any of the input images isn't represented in the coadd (due to
1109  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1110  these pixels.
1111 
1112  Parameters
1113  ----------
1114  mask : `lsst.afw.image.Mask`
1115  Coadded exposure's mask, modified in-place.
1116  """
1117  mask.addMaskPlane("INEXACT_PSF")
1118  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1119  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1120  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1121  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1122  array = mask.getArray()
1123  selected = array & (sensorEdge | clipped | rejected) > 0
1124  array[selected] |= inexactPsf
1125 
1126  @classmethod
1127  def _makeArgumentParser(cls):
1128  """Create an argument parser.
1129  """
1130  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1131  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1132  + cls.ConfigClass().warpType + "Warp",
1133  help="data ID, e.g. --id tract=12345 patch=1,2",
1134  ContainerClass=AssembleCoaddDataIdContainer)
1135  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1136  ContainerClass=SelectDataIdContainer)
1137  return parser
1138 
1139  @staticmethod
1140  def _subBBoxIter(bbox, subregionSize):
1141  """Iterate over subregions of a bbox.
1142 
1143  Parameters
1144  ----------
1145  bbox : `lsst.geom.Box2I`
1146  Bounding box over which to iterate.
1147  subregionSize: `lsst.geom.Extent2I`
1148  Size of sub-bboxes.
1149 
1150  Yields
1151  ------
1152  subBBox : `lsst.geom.Box2I`
1153  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1154  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1155  the edges of ``bbox``, but it will never be empty.
1156  """
1157  if bbox.isEmpty():
1158  raise RuntimeError("bbox %s is empty" % (bbox,))
1159  if subregionSize[0] < 1 or subregionSize[1] < 1:
1160  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1161 
1162  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1163  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1164  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1165  subBBox.clip(bbox)
1166  if subBBox.isEmpty():
1167  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1168  "colShift=%s, rowShift=%s" %
1169  (bbox, subregionSize, colShift, rowShift))
1170  yield subBBox
1171 
1172 
1173 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1174  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1175  """
1176 
1177  def makeDataRefList(self, namespace):
1178  """Make self.refList from self.idList.
1179 
1180  Parameters
1181  ----------
1182  namespace
1183  Results of parsing command-line (with ``butler`` and ``log`` elements).
1184  """
1185  datasetType = namespace.config.coaddName + "Coadd"
1186  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1187 
1188  for dataId in self.idList:
1189  # tract and patch are required
1190  for key in keysCoadd:
1191  if key not in dataId:
1192  raise RuntimeError("--id must include " + key)
1193 
1194  dataRef = namespace.butler.dataRef(
1195  datasetType=datasetType,
1196  dataId=dataId,
1197  )
1198  self.refList.append(dataRef)
1199 
1200 
1201 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1202  """Function to count the number of pixels with a specific mask in a
1203  footprint.
1204 
1205  Find the intersection of mask & footprint. Count all pixels in the mask
1206  that are in the intersection that have bitmask set but do not have
1207  ignoreMask set. Return the count.
1208 
1209  Parameters
1210  ----------
1211  mask : `lsst.afw.image.Mask`
1212  Mask to define intersection region by.
1213  footprint : `lsst.afw.detection.Footprint`
1214  Footprint to define the intersection region by.
1215  bitmask
1216  Specific mask that we wish to count the number of occurances of.
1217  ignoreMask
1218  Pixels to not consider.
1219 
1220  Returns
1221  -------
1222  result : `int`
1223  Count of number of pixels in footprint with specified mask.
1224  """
1225  bbox = footprint.getBBox()
1226  bbox.clip(mask.getBBox(afwImage.PARENT))
1227  fp = afwImage.Mask(bbox)
1228  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1229  footprint.spans.setMask(fp, bitmask)
1230  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1231  (subMask.getArray() & ignoreMask) == 0).sum()
1232 
1233 
1234 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1235  """Configuration parameters for the SafeClipAssembleCoaddTask.
1236  """
1237  assembleMeanCoadd = pexConfig.ConfigurableField(
1238  target=AssembleCoaddTask,
1239  doc="Task to assemble an initial Coadd using the MEAN statistic.",
1240  )
1241  assembleMeanClipCoadd = pexConfig.ConfigurableField(
1242  target=AssembleCoaddTask,
1243  doc="Task to assemble an initial Coadd using the MEANCLIP statistic.",
1244  )
1245  clipDetection = pexConfig.ConfigurableField(
1246  target=SourceDetectionTask,
1247  doc="Detect sources on difference between unclipped and clipped coadd")
1248  minClipFootOverlap = pexConfig.Field(
1249  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1250  dtype=float,
1251  default=0.6
1252  )
1253  minClipFootOverlapSingle = pexConfig.Field(
1254  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1255  "clipped when only one visit overlaps",
1256  dtype=float,
1257  default=0.5
1258  )
1259  minClipFootOverlapDouble = pexConfig.Field(
1260  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1261  "clipped when two visits overlap",
1262  dtype=float,
1263  default=0.45
1264  )
1265  maxClipFootOverlapDouble = pexConfig.Field(
1266  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1267  "considering two visits",
1268  dtype=float,
1269  default=0.15
1270  )
1271  minBigOverlap = pexConfig.Field(
1272  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1273  "when labeling clipped footprints",
1274  dtype=int,
1275  default=100
1276  )
1277 
1278  def setDefaults(self):
1279  """Set default values for clipDetection.
1280 
1281  Notes
1282  -----
1283  The numeric values for these configuration parameters were
1284  empirically determined, future work may further refine them.
1285  """
1286  AssembleCoaddConfig.setDefaults(self)
1287  self.clipDetectionclipDetection.doTempLocalBackground = False
1288  self.clipDetectionclipDetection.reEstimateBackground = False
1289  self.clipDetectionclipDetection.returnOriginalFootprints = False
1290  self.clipDetectionclipDetection.thresholdPolarity = "both"
1291  self.clipDetectionclipDetection.thresholdValue = 2
1292  self.clipDetectionclipDetection.nSigmaToGrow = 2
1293  self.clipDetectionclipDetection.minPixels = 4
1294  self.clipDetectionclipDetection.isotropicGrow = True
1295  self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1296  self.sigmaClipsigmaClip = 1.5
1297  self.clipIterclipIter = 3
1298  self.statisticstatistic = "MEAN"
1299  self.assembleMeanCoaddassembleMeanCoadd.statistic = 'MEAN'
1300  self.assembleMeanClipCoaddassembleMeanClipCoadd.statistic = 'MEANCLIP'
1301  self.assembleMeanCoaddassembleMeanCoadd.doWrite = False
1302  self.assembleMeanClipCoaddassembleMeanClipCoadd.doWrite = False
1303 
1304  def validate(self):
1305  if self.doSigmaClipdoSigmaClip:
1306  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1307  "Ignoring doSigmaClip.")
1308  self.doSigmaClipdoSigmaClip = False
1309  if self.statisticstatistic != "MEAN":
1310  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1311  "(%s chosen). Please set statistic to MEAN."
1312  % (self.statisticstatistic))
1313  AssembleCoaddTask.ConfigClass.validate(self)
1314 
1315 
1316 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1317  """Assemble a coadded image from a set of coadded temporary exposures,
1318  being careful to clip & flag areas with potential artifacts.
1319 
1320  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1321  we clip outliers). The problem with doing this is that when computing the
1322  coadd PSF at a given location, individual visit PSFs from visits with
1323  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1324  In this task, we correct for this behavior by creating a new
1325  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1326  coaddTempExps and the final coadd where
1327 
1328  i. difference imaging suggests that there is an outlier and
1329  ii. this outlier appears on only one or two images.
1330 
1331  Such regions will not contribute to the final coadd. Furthermore, any
1332  routine to determine the coadd PSF can now be cognizant of clipped regions.
1333  Note that the algorithm implemented by this task is preliminary and works
1334  correctly for HSC data. Parameter modifications and or considerable
1335  redesigning of the algorithm is likley required for other surveys.
1336 
1337  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1338  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1339  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1340  if you wish.
1341 
1342  Notes
1343  -----
1344  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1345  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1346  see `baseDebug` for more about ``debug.py`` files.
1347  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1348  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1349  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1350  for further information.
1351 
1352  Examples
1353  --------
1354  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1355  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1356  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1357 
1358  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1359  and filter to be coadded (specified using
1360  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1361  along with a list of coaddTempExps to attempt to coadd (specified using
1362  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1363  Only the coaddTempExps that cover the specified tract and patch will be
1364  coadded. A list of the available optional arguments can be obtained by
1365  calling assembleCoadd.py with the --help command line argument:
1366 
1367  .. code-block:: none
1368 
1369  assembleCoadd.py --help
1370 
1371  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1372  context of multi-band processing, we will generate the HSC-I & -R band
1373  coadds from HSC engineering test data provided in the ci_hsc package.
1374  To begin, assuming that the lsst stack has been already set up, we must
1375  set up the obs_subaru and ci_hsc packages. This defines the environment
1376  variable $CI_HSC_DIR and points at the location of the package. The raw
1377  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1378  the coadds, we must first
1379 
1380  - ``processCcd``
1381  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1382  - ``makeSkyMap``
1383  create a skymap that covers the area of the sky present in the raw exposures
1384  - ``makeCoaddTempExp``
1385  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1386 
1387  We can perform all of these steps by running
1388 
1389  .. code-block:: none
1390 
1391  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1392 
1393  This will produce warped coaddTempExps for each visit. To coadd the
1394  warped data, we call ``assembleCoadd.py`` as follows:
1395 
1396  .. code-block:: none
1397 
1398  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1399  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1400  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1401  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1402  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1403  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1404  --selectId visit=903988 ccd=24
1405 
1406  This will process the HSC-I band data. The results are written in
1407  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1408 
1409  You may also choose to run:
1410 
1411  .. code-block:: none
1412 
1413  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1414  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1415  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1416  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1417  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1418  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1419  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1420  --selectId visit=903346 ccd=12
1421 
1422  to generate the coadd for the HSC-R band if you are interested in following
1423  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1424  """
1425  ConfigClass = SafeClipAssembleCoaddConfig
1426  _DefaultName = "safeClipAssembleCoadd"
1427 
1428  def __init__(self, *args, **kwargs):
1429  AssembleCoaddTask.__init__(self, *args, **kwargs)
1430  schema = afwTable.SourceTable.makeMinimalSchema()
1431  self.makeSubtask("clipDetection", schema=schema)
1432  self.makeSubtask("assembleMeanClipCoadd")
1433  self.makeSubtask("assembleMeanCoadd")
1434 
1435  @utils.inheritDoc(AssembleCoaddTask)
1436  @pipeBase.timeMethod
1437  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1438  """Assemble the coadd for a region.
1439 
1440  Compute the difference of coadds created with and without outlier
1441  rejection to identify coadd pixels that have outlier values in some
1442  individual visits.
1443  Detect clipped regions on the difference image and mark these regions
1444  on the one or two individual coaddTempExps where they occur if there
1445  is significant overlap between the clipped region and a source. This
1446  leaves us with a set of footprints from the difference image that have
1447  been identified as having occured on just one or two individual visits.
1448  However, these footprints were generated from a difference image. It
1449  is conceivable for a large diffuse source to have become broken up
1450  into multiple footprints acrosss the coadd difference in this process.
1451  Determine the clipped region from all overlapping footprints from the
1452  detected sources in each visit - these are big footprints.
1453  Combine the small and big clipped footprints and mark them on a new
1454  bad mask plane.
1455  Generate the coadd using `AssembleCoaddTask.run` without outlier
1456  removal. Clipped footprints will no longer make it into the coadd
1457  because they are marked in the new bad mask plane.
1458 
1459  Notes
1460  -----
1461  args and kwargs are passed but ignored in order to match the call
1462  signature expected by the parent task.
1463  """
1464  exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1465  mask = exp.getMaskedImage().getMask()
1466  mask.addMaskPlane("CLIPPED")
1467 
1468  result = self.detectClipdetectClip(exp, tempExpRefList)
1469 
1470  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1471 
1472  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1473  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1474  # Append big footprints from individual Warps to result.clipSpans
1475  bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1476  result.detectionFootprints, maskClipValue, maskDetValue,
1477  exp.getBBox())
1478  # Create mask of the current clipped footprints
1479  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1480  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1481 
1482  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1483  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1484  maskClip |= maskClipBig
1485 
1486  # Assemble coadd from base class, but ignoring CLIPPED pixels
1487  badMaskPlanes = self.config.badMaskPlanes[:]
1488  badMaskPlanes.append("CLIPPED")
1489  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1490  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1491  result.clipSpans, mask=badPixelMask)
1492 
1493  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1494  """Return an exposure that contains the difference between unclipped
1495  and clipped coadds.
1496 
1497  Generate a difference image between clipped and unclipped coadds.
1498  Compute the difference image by subtracting an outlier-clipped coadd
1499  from an outlier-unclipped coadd. Return the difference image.
1500 
1501  Parameters
1502  ----------
1503  skyInfo : `lsst.pipe.base.Struct`
1504  Patch geometry information, from getSkyInfo
1505  tempExpRefList : `list`
1506  List of data reference to tempExp
1507  imageScalerList : `list`
1508  List of image scalers
1509  weightList : `list`
1510  List of weights
1511 
1512  Returns
1513  -------
1514  exp : `lsst.afw.image.Exposure`
1515  Difference image of unclipped and clipped coadd wrapped in an Exposure
1516  """
1517  coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList,
1518  imageScalerList, weightList).coaddExposure
1519 
1520  coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList,
1521  imageScalerList, weightList).coaddExposure
1522 
1523  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1524  coaddDiff -= coaddClip.getMaskedImage()
1525  exp = afwImage.ExposureF(coaddDiff)
1526  exp.setPsf(coaddMean.getPsf())
1527  return exp
1528 
1529  def detectClip(self, exp, tempExpRefList):
1530  """Detect clipped regions on an exposure and set the mask on the
1531  individual tempExp masks.
1532 
1533  Detect footprints in the difference image after smoothing the
1534  difference image with a Gaussian kernal. Identify footprints that
1535  overlap with one or two input ``coaddTempExps`` by comparing the
1536  computed overlap fraction to thresholds set in the config. A different
1537  threshold is applied depending on the number of overlapping visits
1538  (restricted to one or two). If the overlap exceeds the thresholds,
1539  the footprint is considered "CLIPPED" and is marked as such on the
1540  coaddTempExp. Return a struct with the clipped footprints, the indices
1541  of the ``coaddTempExps`` that end up overlapping with the clipped
1542  footprints, and a list of new masks for the ``coaddTempExps``.
1543 
1544  Parameters
1545  ----------
1546  exp : `lsst.afw.image.Exposure`
1547  Exposure to run detection on.
1548  tempExpRefList : `list`
1549  List of data reference to tempExp.
1550 
1551  Returns
1552  -------
1553  result : `lsst.pipe.base.Struct`
1554  Result struct with components:
1555 
1556  - ``clipFootprints``: list of clipped footprints.
1557  - ``clipIndices``: indices for each ``clippedFootprint`` in
1558  ``tempExpRefList``.
1559  - ``clipSpans``: List of dictionaries containing spanSet lists
1560  to clip. Each element contains the new maskplane name
1561  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1562  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1563  compressed into footprints.
1564  """
1565  mask = exp.getMaskedImage().getMask()
1566  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1567  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1568  # Merge positive and negative together footprints together
1569  fpSet.positive.merge(fpSet.negative)
1570  footprints = fpSet.positive
1571  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1572  ignoreMask = self.getBadPixelMask()
1573 
1574  clipFootprints = []
1575  clipIndices = []
1576  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1577 
1578  # for use by detectClipBig
1579  visitDetectionFootprints = []
1580 
1581  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1582  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1583  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1584 
1585  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1586  for i, warpRef in enumerate(tempExpRefList):
1587  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1588  immediate=True).getMaskedImage().getMask()
1589  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1590  afwImage.PARENT, True)
1591  maskVisitDet &= maskDetValue
1592  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1593  visitDetectionFootprints.append(visitFootprints)
1594 
1595  for j, footprint in enumerate(footprints.getFootprints()):
1596  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1597  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1598 
1599  # build a list of clipped spans for each visit
1600  for j, footprint in enumerate(footprints.getFootprints()):
1601  nPixel = footprint.getArea()
1602  overlap = [] # hold the overlap with each visit
1603  indexList = [] # index of visit in global list
1604  for i in range(len(tempExpRefList)):
1605  ignore = ignoreArr[i, j]
1606  overlapDet = overlapDetArr[i, j]
1607  totPixel = nPixel - ignore
1608 
1609  # If we have more bad pixels than detection skip
1610  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1611  continue
1612  overlap.append(overlapDet/float(totPixel))
1613  indexList.append(i)
1614 
1615  overlap = numpy.array(overlap)
1616  if not len(overlap):
1617  continue
1618 
1619  keep = False # Should this footprint be marked as clipped?
1620  keepIndex = [] # Which tempExps does the clipped footprint belong to
1621 
1622  # If footprint only has one overlap use a lower threshold
1623  if len(overlap) == 1:
1624  if overlap[0] > self.config.minClipFootOverlapSingle:
1625  keep = True
1626  keepIndex = [0]
1627  else:
1628  # This is the general case where only visit should be clipped
1629  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1630  if len(clipIndex) == 1:
1631  keep = True
1632  keepIndex = [clipIndex[0]]
1633 
1634  # Test if there are clipped objects that overlap two different visits
1635  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1636  if len(clipIndex) == 2 and len(overlap) > 3:
1637  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1638  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1639  keep = True
1640  keepIndex = clipIndex
1641 
1642  if not keep:
1643  continue
1644 
1645  for index in keepIndex:
1646  globalIndex = indexList[index]
1647  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1648 
1649  clipIndices.append(numpy.array(indexList)[keepIndex])
1650  clipFootprints.append(footprint)
1651 
1652  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1653  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1654 
1655  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1656  maskClipValue, maskDetValue, coaddBBox):
1657  """Return individual warp footprints for large artifacts and append
1658  them to ``clipList`` in place.
1659 
1660  Identify big footprints composed of many sources in the coadd
1661  difference that may have originated in a large diffuse source in the
1662  coadd. We do this by indentifying all clipped footprints that overlap
1663  significantly with each source in all the coaddTempExps.
1664 
1665  Parameters
1666  ----------
1667  clipList : `list`
1668  List of alt mask SpanSets with clipping information. Modified.
1669  clipFootprints : `list`
1670  List of clipped footprints.
1671  clipIndices : `list`
1672  List of which entries in tempExpClipList each footprint belongs to.
1673  maskClipValue
1674  Mask value of clipped pixels.
1675  maskDetValue
1676  Mask value of detected pixels.
1677  coaddBBox : `lsst.geom.Box`
1678  BBox of the coadd and warps.
1679 
1680  Returns
1681  -------
1682  bigFootprintsCoadd : `list`
1683  List of big footprints
1684  """
1685  bigFootprintsCoadd = []
1686  ignoreMask = self.getBadPixelMask()
1687  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1688  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1689  for footprint in visitFootprints.getFootprints():
1690  footprint.spans.setMask(maskVisitDet, maskDetValue)
1691 
1692  # build a mask of clipped footprints that are in this visit
1693  clippedFootprintsVisit = []
1694  for foot, clipIndex in zip(clipFootprints, clipIndices):
1695  if index not in clipIndex:
1696  continue
1697  clippedFootprintsVisit.append(foot)
1698  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1699  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1700 
1701  bigFootprintsVisit = []
1702  for foot in visitFootprints.getFootprints():
1703  if foot.getArea() < self.config.minBigOverlap:
1704  continue
1705  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1706  if nCount > self.config.minBigOverlap:
1707  bigFootprintsVisit.append(foot)
1708  bigFootprintsCoadd.append(foot)
1709 
1710  for footprint in bigFootprintsVisit:
1711  clippedSpans["CLIPPED"].append(footprint.spans)
1712 
1713  return bigFootprintsCoadd
1714 
1715 
1717  psfMatchedWarps = pipeBase.connectionTypes.Input(
1718  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1719  "Only PSF-Matched Warps make sense for image subtraction. "
1720  "Therefore, they must be an additional declared input."),
1721  name="{inputCoaddName}Coadd_psfMatchedWarp",
1722  storageClass="ExposureF",
1723  dimensions=("tract", "patch", "skymap", "visit"),
1724  deferLoad=True,
1725  multiple=True
1726  )
1727  templateCoadd = pipeBase.connectionTypes.Output(
1728  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1729  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1730  name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1731  storageClass="ExposureF",
1732  dimensions=("tract", "patch", "skymap", "band"),
1733  )
1734 
1735  def __init__(self, *, config=None):
1736  super().__init__(config=config)
1737  if not config.assembleStaticSkyModel.doWrite:
1738  self.outputs.remove("templateCoadd")
1739  config.validate()
1740 
1741 
1742 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1743  pipelineConnections=CompareWarpAssembleCoaddConnections):
1744  assembleStaticSkyModel = pexConfig.ConfigurableField(
1745  target=AssembleCoaddTask,
1746  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1747  " naive/first-iteration model of the static sky.",
1748  )
1749  detect = pexConfig.ConfigurableField(
1750  target=SourceDetectionTask,
1751  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1752  )
1753  detectTemplate = pexConfig.ConfigurableField(
1754  target=SourceDetectionTask,
1755  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1756  )
1757  maskStreaks = pexConfig.ConfigurableField(
1758  target=MaskStreaksTask,
1759  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1760  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1761  "streakMaskName"
1762  )
1763  streakMaskName = pexConfig.Field(
1764  dtype=str,
1765  default="STREAK",
1766  doc="Name of mask bit used for streaks"
1767  )
1768  maxNumEpochs = pexConfig.Field(
1769  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1770  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1771  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1772  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1773  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1774  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1775  "than transient and not masked.",
1776  dtype=int,
1777  default=2
1778  )
1779  maxFractionEpochsLow = pexConfig.RangeField(
1780  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1781  "Effective maxNumEpochs = "
1782  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1783  dtype=float,
1784  default=0.4,
1785  min=0., max=1.,
1786  )
1787  maxFractionEpochsHigh = pexConfig.RangeField(
1788  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1789  "Effective maxNumEpochs = "
1790  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1791  dtype=float,
1792  default=0.03,
1793  min=0., max=1.,
1794  )
1795  spatialThreshold = pexConfig.RangeField(
1796  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1797  "temporal criteria. If 0, clip all. If 1, clip none.",
1798  dtype=float,
1799  default=0.5,
1800  min=0., max=1.,
1801  inclusiveMin=True, inclusiveMax=True
1802  )
1803  doScaleWarpVariance = pexConfig.Field(
1804  doc="Rescale Warp variance plane using empirical noise?",
1805  dtype=bool,
1806  default=True,
1807  )
1808  scaleWarpVariance = pexConfig.ConfigurableField(
1809  target=ScaleVarianceTask,
1810  doc="Rescale variance on warps",
1811  )
1812  doPreserveContainedBySource = pexConfig.Field(
1813  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1814  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1815  dtype=bool,
1816  default=True,
1817  )
1818  doPrefilterArtifacts = pexConfig.Field(
1819  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1820  "because they will be excluded anyway. This prevents them from contributing "
1821  "to the outlier epoch count image and potentially being labeled as persistant."
1822  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1823  dtype=bool,
1824  default=True
1825  )
1826  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1827  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1828  dtype=str,
1829  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1830  )
1831  prefilterArtifactsRatio = pexConfig.Field(
1832  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1833  dtype=float,
1834  default=0.05
1835  )
1836  doFilterMorphological = pexConfig.Field(
1837  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1838  "be streaks.",
1839  dtype=bool,
1840  default=False
1841  )
1842 
1843  def setDefaults(self):
1844  AssembleCoaddConfig.setDefaults(self)
1845  self.statisticstatistic = 'MEAN'
1846  self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
1847 
1848  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1849  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1850  if "EDGE" in self.badMaskPlanes:
1851  self.badMaskPlanes.remove('EDGE')
1852  self.removeMaskPlanes.append('EDGE')
1853  self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1854  self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
1855  self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
1856  self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
1857  self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
1858  self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
1859  self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
1860  self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
1861  self.detectdetect.doTempLocalBackground = False
1862  self.detectdetect.reEstimateBackground = False
1863  self.detectdetect.returnOriginalFootprints = False
1864  self.detectdetect.thresholdPolarity = "both"
1865  self.detectdetect.thresholdValue = 5
1866  self.detectdetect.minPixels = 4
1867  self.detectdetect.isotropicGrow = True
1868  self.detectdetect.thresholdType = "pixel_stdev"
1869  self.detectdetect.nSigmaToGrow = 0.4
1870  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1871  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1872  self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
1873  self.detectTemplatedetectTemplate.doTempLocalBackground = False
1874  self.detectTemplatedetectTemplate.reEstimateBackground = False
1875  self.detectTemplatedetectTemplate.returnOriginalFootprints = False
1876 
1877  def validate(self):
1878  super().validate()
1879  if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
1880  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1881  "Please set assembleStaticSkyModel.doNImage=False")
1882 
1883  if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
1884  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1885  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1886  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1887  "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
1888 
1889 
1890 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1891  """Assemble a compareWarp coadded image from a set of warps
1892  by masking artifacts detected by comparing PSF-matched warps.
1893 
1894  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1895  we clip outliers). The problem with doing this is that when computing the
1896  coadd PSF at a given location, individual visit PSFs from visits with
1897  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1898  In this task, we correct for this behavior by creating a new badMaskPlane
1899  'CLIPPED' which marks pixels in the individual warps suspected to contain
1900  an artifact. We populate this plane on the input warps by comparing
1901  PSF-matched warps with a PSF-matched median coadd which serves as a
1902  model of the static sky. Any group of pixels that deviates from the
1903  PSF-matched template coadd by more than config.detect.threshold sigma,
1904  is an artifact candidate. The candidates are then filtered to remove
1905  variable sources and sources that are difficult to subtract such as
1906  bright stars. This filter is configured using the config parameters
1907  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1908  the maximum fraction of epochs that the deviation can appear in and still
1909  be considered an artifact. The spatialThreshold is the maximum fraction of
1910  pixels in the footprint of the deviation that appear in other epochs
1911  (where other epochs is defined by the temporalThreshold). If the deviant
1912  region meets this criteria of having a significant percentage of pixels
1913  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1914  set in the mask. These regions will not contribute to the final coadd.
1915  Furthermore, any routine to determine the coadd PSF can now be cognizant
1916  of clipped regions. Note that the algorithm implemented by this task is
1917  preliminary and works correctly for HSC data. Parameter modifications and
1918  or considerable redesigning of the algorithm is likley required for other
1919  surveys.
1920 
1921  ``CompareWarpAssembleCoaddTask`` sub-classes
1922  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1923  as a subtask to generate the TemplateCoadd (the model of the static sky).
1924 
1925  Notes
1926  -----
1927  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1928  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1929  ``baseDebug`` for more about ``debug.py`` files.
1930 
1931  This task supports the following debug variables:
1932 
1933  - ``saveCountIm``
1934  If True then save the Epoch Count Image as a fits file in the `figPath`
1935  - ``figPath``
1936  Path to save the debug fits images and figures
1937 
1938  For example, put something like:
1939 
1940  .. code-block:: python
1941 
1942  import lsstDebug
1943  def DebugInfo(name):
1944  di = lsstDebug.getInfo(name)
1945  if name == "lsst.pipe.tasks.assembleCoadd":
1946  di.saveCountIm = True
1947  di.figPath = "/desired/path/to/debugging/output/images"
1948  return di
1949  lsstDebug.Info = DebugInfo
1950 
1951  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1952  ``--debug`` flag. Some subtasks may have their own debug variables;
1953  see individual Task documentation.
1954 
1955  Examples
1956  --------
1957  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1958  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1959  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1960  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1961  and filter to be coadded (specified using
1962  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1963  along with a list of coaddTempExps to attempt to coadd (specified using
1964  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1965  Only the warps that cover the specified tract and patch will be coadded.
1966  A list of the available optional arguments can be obtained by calling
1967  ``assembleCoadd.py`` with the ``--help`` command line argument:
1968 
1969  .. code-block:: none
1970 
1971  assembleCoadd.py --help
1972 
1973  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1974  context of multi-band processing, we will generate the HSC-I & -R band
1975  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1976  To begin, assuming that the lsst stack has been already set up, we must
1977  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1978  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1979  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1980  directory. To begin assembling the coadds, we must first
1981 
1982  - processCcd
1983  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1984  - makeSkyMap
1985  create a skymap that covers the area of the sky present in the raw exposures
1986  - makeCoaddTempExp
1987  warp the individual calibrated exposures to the tangent plane of the coadd
1988 
1989  We can perform all of these steps by running
1990 
1991  .. code-block:: none
1992 
1993  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1994 
1995  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1996  warped data, we call ``assembleCoadd.py`` as follows:
1997 
1998  .. code-block:: none
1999 
2000  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2001  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2002  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2003  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2004  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2005  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2006  --selectId visit=903988 ccd=24
2007 
2008  This will process the HSC-I band data. The results are written in
2009  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2010  """
2011  ConfigClass = CompareWarpAssembleCoaddConfig
2012  _DefaultName = "compareWarpAssembleCoadd"
2013 
2014  def __init__(self, *args, **kwargs):
2015  AssembleCoaddTask.__init__(self, *args, **kwargs)
2016  self.makeSubtask("assembleStaticSkyModel")
2017  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2018  self.makeSubtask("detect", schema=detectionSchema)
2019  if self.config.doPreserveContainedBySource:
2020  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2021  if self.config.doScaleWarpVariance:
2022  self.makeSubtask("scaleWarpVariance")
2023  if self.config.doFilterMorphological:
2024  self.makeSubtask("maskStreaks")
2025 
2026  @utils.inheritDoc(AssembleCoaddTask)
2027  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2028  """
2029  Generate a templateCoadd to use as a naive model of static sky to
2030  subtract from PSF-Matched warps.
2031 
2032  Returns
2033  -------
2034  result : `lsst.pipe.base.Struct`
2035  Result struct with components:
2036 
2037  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2038  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2039  """
2040  # Ensure that psfMatchedWarps are used as input warps for template generation
2041  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2042  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2043 
2044  # Because subtasks don't have connections we have to make one.
2045  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2046  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2047  if self.config.assembleStaticSkyModel.doWrite:
2048  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2049  # Remove template coadd from both subtask's and main tasks outputs,
2050  # because it is handled by the subtask as `coaddExposure`
2051  del outputRefs.templateCoadd
2052  del staticSkyModelOutputRefs.templateCoadd
2053 
2054  # A PSF-Matched nImage does not exist as a dataset type
2055  if 'nImage' in staticSkyModelOutputRefs.keys():
2056  del staticSkyModelOutputRefs.nImage
2057 
2058  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2059  staticSkyModelOutputRefs)
2060  if templateCoadd is None:
2061  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2062 
2063  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2064  nImage=templateCoadd.nImage,
2065  warpRefList=templateCoadd.warpRefList,
2066  imageScalerList=templateCoadd.imageScalerList,
2067  weightList=templateCoadd.weightList)
2068 
2069  @utils.inheritDoc(AssembleCoaddTask)
2070  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2071  """
2072  Generate a templateCoadd to use as a naive model of static sky to
2073  subtract from PSF-Matched warps.
2074 
2075  Returns
2076  -------
2077  result : `lsst.pipe.base.Struct`
2078  Result struct with components:
2079 
2080  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2081  - ``nImage``: N Image (``lsst.afw.image.Image``)
2082  """
2083  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2084  if templateCoadd is None:
2085  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2086 
2087  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2088  nImage=templateCoadd.nImage,
2089  warpRefList=templateCoadd.warpRefList,
2090  imageScalerList=templateCoadd.imageScalerList,
2091  weightList=templateCoadd.weightList)
2092 
2093  def _noTemplateMessage(self, warpType):
2094  warpName = (warpType[0].upper() + warpType[1:])
2095  message = """No %(warpName)s warps were found to build the template coadd which is
2096  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2097  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2098  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2099 
2100  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2101  another algorithm like:
2102 
2103  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2104  config.assemble.retarget(SafeClipAssembleCoaddTask)
2105  """ % {"warpName": warpName}
2106  return message
2107 
2108  @utils.inheritDoc(AssembleCoaddTask)
2109  @pipeBase.timeMethod
2110  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2111  supplementaryData, *args, **kwargs):
2112  """Assemble the coadd.
2113 
2114  Find artifacts and apply them to the warps' masks creating a list of
2115  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2116  plane. Then pass these alternative masks to the base class's `run`
2117  method.
2118 
2119  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2120  that must contain a ``templateCoadd`` that serves as the
2121  model of the static sky.
2122  """
2123 
2124  # Check and match the order of the supplementaryData
2125  # (PSF-matched) inputs to the order of the direct inputs,
2126  # so that the artifact mask is applied to the right warp
2127  dataIds = [ref.dataId for ref in tempExpRefList]
2128  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2129 
2130  if dataIds != psfMatchedDataIds:
2131  self.log.info("Reordering and or/padding PSF-matched visit input list")
2132  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2133  psfMatchedDataIds, dataIds)
2134  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2135  psfMatchedDataIds, dataIds)
2136 
2137  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2138  spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2139  supplementaryData.warpRefList,
2140  supplementaryData.imageScalerList)
2141 
2142  badMaskPlanes = self.config.badMaskPlanes[:]
2143  badMaskPlanes.append("CLIPPED")
2144  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2145 
2146  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2147  spanSetMaskList, mask=badPixelMask)
2148 
2149  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2150  # Psf-Matching moves the real edge inwards
2151  self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2152  return result
2153 
2154  def applyAltEdgeMask(self, mask, altMaskList):
2155  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2156 
2157  Parameters
2158  ----------
2159  mask : `lsst.afw.image.Mask`
2160  Original mask.
2161  altMaskList : `list`
2162  List of Dicts containing ``spanSet`` lists.
2163  Each element contains the new mask plane name (e.g. "CLIPPED
2164  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2165  the mask.
2166  """
2167  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2168  for visitMask in altMaskList:
2169  if "EDGE" in visitMask:
2170  for spanSet in visitMask['EDGE']:
2171  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2172 
2173  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2174  """Find artifacts.
2175 
2176  Loop through warps twice. The first loop builds a map with the count
2177  of how many epochs each pixel deviates from the templateCoadd by more
2178  than ``config.chiThreshold`` sigma. The second loop takes each
2179  difference image and filters the artifacts detected in each using
2180  count map to filter out variable sources and sources that are
2181  difficult to subtract cleanly.
2182 
2183  Parameters
2184  ----------
2185  templateCoadd : `lsst.afw.image.Exposure`
2186  Exposure to serve as model of static sky.
2187  tempExpRefList : `list`
2188  List of data references to warps.
2189  imageScalerList : `list`
2190  List of image scalers.
2191 
2192  Returns
2193  -------
2194  altMasks : `list`
2195  List of dicts containing information about CLIPPED
2196  (i.e., artifacts), NO_DATA, and EDGE pixels.
2197  """
2198 
2199  self.log.debug("Generating Count Image, and mask lists.")
2200  coaddBBox = templateCoadd.getBBox()
2201  slateIm = afwImage.ImageU(coaddBBox)
2202  epochCountImage = afwImage.ImageU(coaddBBox)
2203  nImage = afwImage.ImageU(coaddBBox)
2204  spanSetArtifactList = []
2205  spanSetNoDataMaskList = []
2206  spanSetEdgeList = []
2207  spanSetBadMorphoList = []
2208  badPixelMask = self.getBadPixelMask()
2209 
2210  # mask of the warp diffs should = that of only the warp
2211  templateCoadd.mask.clearAllMaskPlanes()
2212 
2213  if self.config.doPreserveContainedBySource:
2214  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2215  else:
2216  templateFootprints = None
2217 
2218  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2219  warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2220  if warpDiffExp is not None:
2221  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2222  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2223  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2224  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2225  fpSet.positive.merge(fpSet.negative)
2226  footprints = fpSet.positive
2227  slateIm.set(0)
2228  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2229 
2230  # Remove artifacts due to defects before they contribute to the epochCountImage
2231  if self.config.doPrefilterArtifacts:
2232  spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2233 
2234  # Clear mask before adding prefiltered spanSets
2235  self.detect.clearMask(warpDiffExp.mask)
2236  for spans in spanSetList:
2237  spans.setImage(slateIm, 1, doClip=True)
2238  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2239  epochCountImage += slateIm
2240 
2241  if self.config.doFilterMorphological:
2242  maskName = self.config.streakMaskName
2243  _ = self.maskStreaks.run(warpDiffExp)
2244  streakMask = warpDiffExp.mask
2245  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2246  streakMask.getPlaneBitMask(maskName)).split()
2247 
2248  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2249  # undergo a second convolution. Pixels with data in the direct warp
2250  # but not in the PSF-matched warp will not have their artifacts detected.
2251  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2252  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2253  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2254  nansMask.setXY0(warpDiffExp.getXY0())
2255  edgeMask = warpDiffExp.mask
2256  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2257  edgeMask.getPlaneBitMask("EDGE")).split()
2258  else:
2259  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2260  # In this case, mask the whole epoch
2261  nansMask = afwImage.MaskX(coaddBBox, 1)
2262  spanSetList = []
2263  spanSetEdgeMask = []
2264  spanSetStreak = []
2265 
2266  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2267 
2268  spanSetNoDataMaskList.append(spanSetNoDataMask)
2269  spanSetArtifactList.append(spanSetList)
2270  spanSetEdgeList.append(spanSetEdgeMask)
2271  if self.config.doFilterMorphological:
2272  spanSetBadMorphoList.append(spanSetStreak)
2273 
2274  if lsstDebug.Info(__name__).saveCountIm:
2275  path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2276  epochCountImage.writeFits(path)
2277 
2278  for i, spanSetList in enumerate(spanSetArtifactList):
2279  if spanSetList:
2280  filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2281  templateFootprints)
2282  spanSetArtifactList[i] = filteredSpanSetList
2283  if self.config.doFilterMorphological:
2284  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2285 
2286  altMasks = []
2287  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2288  altMasks.append({'CLIPPED': artifacts,
2289  'NO_DATA': noData,
2290  'EDGE': edge})
2291  return altMasks
2292 
2293  def prefilterArtifacts(self, spanSetList, exp):
2294  """Remove artifact candidates covered by bad mask plane.
2295 
2296  Any future editing of the candidate list that does not depend on
2297  temporal information should go in this method.
2298 
2299  Parameters
2300  ----------
2301  spanSetList : `list`
2302  List of SpanSets representing artifact candidates.
2303  exp : `lsst.afw.image.Exposure`
2304  Exposure containing mask planes used to prefilter.
2305 
2306  Returns
2307  -------
2308  returnSpanSetList : `list`
2309  List of SpanSets with artifacts.
2310  """
2311  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2312  goodArr = (exp.mask.array & badPixelMask) == 0
2313  returnSpanSetList = []
2314  bbox = exp.getBBox()
2315  x0, y0 = exp.getXY0()
2316  for i, span in enumerate(spanSetList):
2317  y, x = span.clippedTo(bbox).indices()
2318  yIndexLocal = numpy.array(y) - y0
2319  xIndexLocal = numpy.array(x) - x0
2320  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2321  if goodRatio > self.config.prefilterArtifactsRatio:
2322  returnSpanSetList.append(span)
2323  return returnSpanSetList
2324 
2325  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2326  """Filter artifact candidates.
2327 
2328  Parameters
2329  ----------
2330  spanSetList : `list`
2331  List of SpanSets representing artifact candidates.
2332  epochCountImage : `lsst.afw.image.Image`
2333  Image of accumulated number of warpDiff detections.
2334  nImage : `lsst.afw.image.Image`
2335  Image of the accumulated number of total epochs contributing.
2336 
2337  Returns
2338  -------
2339  maskSpanSetList : `list`
2340  List of SpanSets with artifacts.
2341  """
2342 
2343  maskSpanSetList = []
2344  x0, y0 = epochCountImage.getXY0()
2345  for i, span in enumerate(spanSetList):
2346  y, x = span.indices()
2347  yIdxLocal = [y1 - y0 for y1 in y]
2348  xIdxLocal = [x1 - x0 for x1 in x]
2349  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2350  totalN = nImage.array[yIdxLocal, xIdxLocal]
2351 
2352  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2353  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2354  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2355  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2356  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2357  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2358  & (outlierN <= effectiveMaxNumEpochs))
2359  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2360  if percentBelowThreshold > self.config.spatialThreshold:
2361  maskSpanSetList.append(span)
2362 
2363  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2364  # If a candidate is contained by a footprint on the template coadd, do not clip
2365  filteredMaskSpanSetList = []
2366  for span in maskSpanSetList:
2367  doKeep = True
2368  for footprint in footprintsToExclude.positive.getFootprints():
2369  if footprint.spans.contains(span):
2370  doKeep = False
2371  break
2372  if doKeep:
2373  filteredMaskSpanSetList.append(span)
2374  maskSpanSetList = filteredMaskSpanSetList
2375 
2376  return maskSpanSetList
2377 
2378  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2379  """Fetch a warp from the butler and return a warpDiff.
2380 
2381  Parameters
2382  ----------
2383  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2384  Butler dataRef for the warp.
2385  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2386  An image scaler object.
2387  templateCoadd : `lsst.afw.image.Exposure`
2388  Exposure to be substracted from the scaled warp.
2389 
2390  Returns
2391  -------
2392  warp : `lsst.afw.image.Exposure`
2393  Exposure of the image difference between the warp and template.
2394  """
2395 
2396  # If the PSF-Matched warp did not exist for this direct warp
2397  # None is holding its place to maintain order in Gen 3
2398  if warpRef is None:
2399  return None
2400  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2401  warpName = self.getTempExpDatasetName('psfMatched')
2402  if not isinstance(warpRef, DeferredDatasetHandle):
2403  if not warpRef.datasetExists(warpName):
2404  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2405  return None
2406  warp = warpRef.get(datasetType=warpName, immediate=True)
2407  # direct image scaler OK for PSF-matched Warp
2408  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2409  mi = warp.getMaskedImage()
2410  if self.config.doScaleWarpVariance:
2411  try:
2412  self.scaleWarpVariance.run(mi)
2413  except Exception as exc:
2414  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2415  mi -= templateCoadd.getMaskedImage()
2416  return warp
2417 
2418  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2419  """Return a path to which to write debugging output.
2420 
2421  Creates a hyphen-delimited string of dataId values for simple filenames.
2422 
2423  Parameters
2424  ----------
2425  prefix : `str`
2426  Prefix for filename.
2427  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2428  Butler dataRef to make the path from.
2429  coaddLevel : `bool`, optional.
2430  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2431  'filter', but no 'visit').
2432 
2433  Returns
2434  -------
2435  result : `str`
2436  Path for debugging output.
2437  """
2438  if coaddLevel:
2439  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2440  else:
2441  keys = warpRef.dataId.keys()
2442  keyList = sorted(keys, reverse=True)
2443  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2444  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2445  return os.path.join(directory, filename)
2446 
2447 
2448 def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2449  """Match the order of one list to another, padding if necessary
2450 
2451  Parameters
2452  ----------
2453  inputList : list
2454  List to be reordered and padded. Elements can be any type.
2455  inputKeys : iterable
2456  Iterable of values to be compared with outputKeys.
2457  Length must match `inputList`
2458  outputKeys : iterable
2459  Iterable of values to be compared with inputKeys.
2460  padWith :
2461  Any value to be inserted where inputKey not in outputKeys
2462 
2463  Returns
2464  -------
2465  list
2466  Copy of inputList reordered per outputKeys and padded with `padWith`
2467  so that the length matches length of outputKeys.
2468  """
2469  outputList = []
2470  for d in outputKeys:
2471  if d in inputKeys:
2472  outputList.append(inputList[inputKeys.index(d)])
2473  else:
2474  outputList.append(padWith)
2475  return outputList
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:131
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:336
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60