lsst.pipe.tasks  18.1.0-15-gc153667b+3
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41 from .interpImage import InterpImageTask
42 from .scaleZeroPoint import ScaleZeroPointTask
43 from .coaddHelpers import groupPatchExposures, getGroupDataRef
44 from .scaleVariance import ScaleVarianceTask
45 from lsst.meas.algorithms import SourceDetectionTask
46 from lsst.daf.butler import DeferredDatasetHandle
47 
48 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
49  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
50  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
51 
52 
53 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
54  dimensions=("tract", "patch", "abstract_filter", "skymap"),
55  defaultTemplates={"inputCoaddName": "deep",
56  "outputCoaddName": "deep",
57  "warpType": "direct",
58  "warpTypeSuffix": "",
59  "fakesType": ""}):
60  inputWarps = pipeBase.connectionTypes.Input(
61  doc=("Input list of warps to be assemebled i.e. stacked."
62  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
63  name="{inputCoaddName}Coadd_{warpType}Warp",
64  storageClass="ExposureF",
65  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
66  deferLoad=True,
67  multiple=True
68  )
69  skyMap = pipeBase.connectionTypes.Input(
70  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
71  name="{inputCoaddName}Coadd_skyMap",
72  storageClass="SkyMap",
73  dimensions=("skymap", ),
74  )
75  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
76  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
77  " BRIGHT_OBJECT."),
78  name="brightObjectMask",
79  storageClass="ObjectMaskCatalog",
80  dimensions=("tract", "patch", "skymap", "abstract_filter"),
81  )
82  coaddExposure = pipeBase.connectionTypes.Output(
83  doc="Output coadded exposure, produced by stacking input warps",
84  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
85  storageClass="ExposureF",
86  dimensions=("tract", "patch", "skymap", "abstract_filter"),
87  )
88  nImage = pipeBase.connectionTypes.Output(
89  doc="Output image of number of input images per pixel",
90  name="{outputCoaddName}Coadd_nImage",
91  storageClass="ImageU",
92  dimensions=("tract", "patch", "skymap", "abstract_filter"),
93  )
94 
95  def __init__(self, *, config=None):
96  # Override the connection's name template with config to replicate Gen2 behavior
97  config.connections.warpType = config.warpType
98  config.connections.warpTypeSuffix = makeCoaddSuffix(config.warpType)
99 
100  if config.hasFakes:
101  config.connections.fakesType = "_fakes"
102 
103  super().__init__(config=config)
104 
105  if not config.doMaskBrightObjects:
106  self.prerequisiteInputs.remove("brightObjectMask")
107 
108  if not config.doNImage:
109  self.outputs.remove("nImage")
110 
111 
112 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
113  pipelineConnections=AssembleCoaddConnections):
114  """Configuration parameters for the `AssembleCoaddTask`.
115 
116  Notes
117  -----
118  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
119  only set the bitplane config.brightObjectMaskName. To make this useful you
120  *must* also configure the flags.pixel algorithm, for example by adding
121 
122  .. code-block:: none
123 
124  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
125  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
126 
127  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
128  """
129  warpType = pexConfig.Field(
130  doc="Warp name: one of 'direct' or 'psfMatched'",
131  dtype=str,
132  default="direct",
133  )
134  subregionSize = pexConfig.ListField(
135  dtype=int,
136  doc="Width, height of stack subregion size; "
137  "make small enough that a full stack of images will fit into memory at once.",
138  length=2,
139  default=(2000, 2000),
140  )
141  statistic = pexConfig.Field(
142  dtype=str,
143  doc="Main stacking statistic for aggregating over the epochs.",
144  default="MEANCLIP",
145  )
146  doSigmaClip = pexConfig.Field(
147  dtype=bool,
148  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
149  default=False,
150  )
151  sigmaClip = pexConfig.Field(
152  dtype=float,
153  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
154  default=3.0,
155  )
156  clipIter = pexConfig.Field(
157  dtype=int,
158  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
159  default=2,
160  )
161  calcErrorFromInputVariance = pexConfig.Field(
162  dtype=bool,
163  doc="Calculate coadd variance from input variance by stacking statistic."
164  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
165  default=True,
166  )
167  scaleZeroPoint = pexConfig.ConfigurableField(
168  target=ScaleZeroPointTask,
169  doc="Task to adjust the photometric zero point of the coadd temp exposures",
170  )
171  doInterp = pexConfig.Field(
172  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
173  dtype=bool,
174  default=True,
175  )
176  interpImage = pexConfig.ConfigurableField(
177  target=InterpImageTask,
178  doc="Task to interpolate (and extrapolate) over NaN pixels",
179  )
180  doWrite = pexConfig.Field(
181  doc="Persist coadd?",
182  dtype=bool,
183  default=True,
184  )
185  doNImage = pexConfig.Field(
186  doc="Create image of number of contributing exposures for each pixel",
187  dtype=bool,
188  default=False,
189  )
190  doUsePsfMatchedPolygons = pexConfig.Field(
191  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
192  dtype=bool,
193  default=False,
194  )
195  maskPropagationThresholds = pexConfig.DictField(
196  keytype=str,
197  itemtype=float,
198  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
199  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
200  "would have contributed exceeds this value."),
201  default={"SAT": 0.1},
202  )
203  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
204  doc="Mask planes to remove before coadding")
205  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
206  doc="Set mask and flag bits for bright objects?")
207  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
208  doc="Name of mask bit used for bright objects")
209  coaddPsf = pexConfig.ConfigField(
210  doc="Configuration for CoaddPsf",
211  dtype=measAlg.CoaddPsfConfig,
212  )
213  doAttachTransmissionCurve = pexConfig.Field(
214  dtype=bool, default=False, optional=False,
215  doc=("Attach a piecewise TransmissionCurve for the coadd? "
216  "(requires all input Exposures to have TransmissionCurves).")
217  )
218  hasFakes = pexConfig.Field(
219  dtype=bool,
220  default=False,
221  doc="Should be set to True if fake sources have been inserted into the input data."
222  )
223 
224  def setDefaults(self):
225  super().setDefaults()
226  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
227 
228  def validate(self):
229  super().validate()
230  if self.doPsfMatch:
231  # Backwards compatibility.
232  # Configs do not have loggers
233  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
234  self.warpType = 'psfMatched'
235  if self.doSigmaClip and self.statistic != "MEANCLIP":
236  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
237  self.statistic = "MEANCLIP"
238  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
239  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
240  "compute and set a non-zero coadd variance estimate." % (self.statistic))
241 
242  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
243  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
244  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
245  if str(k) not in unstackableStats]
246  raise ValueError("statistic %s is not allowed. Please choose one of %s."
247  % (self.statistic, stackableStats))
248 
249 
250 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
251  """Assemble a coadded image from a set of warps (coadded temporary exposures).
252 
253  We want to assemble a coadded image from a set of Warps (also called
254  coadded temporary exposures or ``coaddTempExps``).
255  Each input Warp covers a patch on the sky and corresponds to a single
256  run/visit/exposure of the covered patch. We provide the task with a list
257  of Warps (``selectDataList``) from which it selects Warps that cover the
258  specified patch (pointed at by ``dataRef``).
259  Each Warp that goes into a coadd will typically have an independent
260  photometric zero-point. Therefore, we must scale each Warp to set it to
261  a common photometric zeropoint. WarpType may be one of 'direct' or
262  'psfMatched', and the boolean configs `config.makeDirect` and
263  `config.makePsfMatched` set which of the warp types will be coadded.
264  The coadd is computed as a mean with optional outlier rejection.
265  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
266  Finally, Warps can have bad 'NaN' pixels which received no input from the
267  source calExps. We interpolate over these bad (NaN) pixels.
268 
269  `AssembleCoaddTask` uses several sub-tasks. These are
270 
271  - `ScaleZeroPointTask`
272  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
273  - `InterpImageTask`
274  - interpolate across bad pixels (NaN) in the final coadd
275 
276  You can retarget these subtasks if you wish.
277 
278  Notes
279  -----
280  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
281  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
282  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
283  no debug variables of its own. Some of the subtasks may support debug
284  variables. See the documentation for the subtasks for further information.
285 
286  Examples
287  --------
288  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
289  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
290  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
291  inputs: a data reference to the tract patch and filter to be coadded, and
292  a list of Warps to attempt to coadd. These are specified using ``--id`` and
293  ``--selectId``, respectively:
294 
295  .. code-block:: none
296 
297  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
298  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
299 
300  Only the Warps that cover the specified tract and patch will be coadded.
301  A list of the available optional arguments can be obtained by calling
302  ``assembleCoadd.py`` with the ``--help`` command line argument:
303 
304  .. code-block:: none
305 
306  assembleCoadd.py --help
307 
308  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
309  multi-band processing, we will generate the HSC-I & -R band coadds from
310  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
311  assuming that the lsst stack has been already set up, we must set up the
312  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
313  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
314  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
315  coadds, we must first
316 
317  - processCcd
318  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
319  - makeSkyMap
320  - create a skymap that covers the area of the sky present in the raw exposures
321  - makeCoaddTempExp
322  - warp the individual calibrated exposures to the tangent plane of the coadd
323 
324  We can perform all of these steps by running
325 
326  .. code-block:: none
327 
328  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
329 
330  This will produce warped exposures for each visit. To coadd the warped
331  data, we call assembleCoadd.py as follows:
332 
333  .. code-block:: none
334 
335  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
336  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
337  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
338  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
339  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
340  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
341  --selectId visit=903988 ccd=24
342 
343  that will process the HSC-I band data. The results are written in
344  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
345 
346  You may also choose to run:
347 
348  .. code-block:: none
349 
350  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
351  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
352  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
353  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
354  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
355  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
356  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
357  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
358 
359  to generate the coadd for the HSC-R band if you are interested in
360  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
361  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
362  rather than `AssembleCoaddTask` to make the coadd.
363  """
364  ConfigClass = AssembleCoaddConfig
365  _DefaultName = "assembleCoadd"
366 
367  def __init__(self, *args, **kwargs):
368  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
369  if args:
370  argNames = ["config", "name", "parentTask", "log"]
371  kwargs.update({k: v for k, v in zip(argNames, args)})
372  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
373  "PipelineTask will not take positional args" % argNames, FutureWarning)
374 
375  super().__init__(**kwargs)
376  self.makeSubtask("interpImage")
377  self.makeSubtask("scaleZeroPoint")
378 
379  if self.config.doMaskBrightObjects:
380  mask = afwImage.Mask()
381  try:
382  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
383  except pexExceptions.LsstCppException:
384  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
385  mask.getMaskPlaneDict().keys())
386  del mask
387 
388  self.warpType = self.config.warpType
389 
390  @utils.inheritDoc(pipeBase.PipelineTask)
391  def runQuantum(self, butlerQC, inputRefs, outputRefs):
392  # Docstring to be formatted with info from PipelineTask.runQuantum
393  """
394  Notes
395  -----
396  Assemble a coadd from a set of Warps.
397 
398  PipelineTask (Gen3) entry point to Coadd a set of Warps.
399  Analogous to `runDataRef`, it prepares all the data products to be
400  passed to `run`, and processes the results before returning a struct
401  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
402  Therefore, its inputs are accessed subregion by subregion
403  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
404  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
405  correspond to an update in `runDataRef` while both entry points
406  are used.
407  """
408  inputData = butlerQC.get(inputRefs)
409 
410  # Construct skyInfo expected by run
411  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
412  skyMap = inputData["skyMap"]
413  outputDataId = butlerQC.quantum.dataId
414 
415  inputData['skyInfo'] = makeSkyInfo(skyMap,
416  tractId=outputDataId['tract'],
417  patchId=outputDataId['patch'])
418 
419  # Construct list of input Deferred Datasets
420  # These quack a bit like like Gen2 DataRefs
421  warpRefList = inputData['inputWarps']
422  # Perform same middle steps as `runDataRef` does
423  inputs = self.prepareInputs(warpRefList)
424  self.log.info("Found %d %s", len(inputs.tempExpRefList),
425  self.getTempExpDatasetName(self.warpType))
426  if len(inputs.tempExpRefList) == 0:
427  self.log.warn("No coadd temporary exposures found")
428  return
429 
430  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
431  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
432  inputs.weightList, supplementaryData=supplementaryData)
433 
434  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
435 
436  if self.config.doWrite:
437  butlerQC.put(retStruct, outputRefs)
438  return retStruct
439 
440  @pipeBase.timeMethod
441  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
442  """Assemble a coadd from a set of Warps.
443 
444  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
445  Compute weights to be applied to each Warp and
446  find scalings to match the photometric zeropoint to a reference Warp.
447  Assemble the Warps using `run`. Interpolate over NaNs and
448  optionally write the coadd to disk. Return the coadded exposure.
449 
450  Parameters
451  ----------
452  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
453  Data reference defining the patch for coaddition and the
454  reference Warp (if ``config.autoReference=False``).
455  Used to access the following data products:
456  - ``self.config.coaddName + "Coadd_skyMap"``
457  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
458  - ``self.config.coaddName + "Coadd"``
459  selectDataList : `list`
460  List of data references to Calexps. Data to be coadded will be
461  selected from this list based on overlap with the patch defined
462  by dataRef, grouped by visit, and converted to a list of data
463  references to warps.
464  warpRefList : `list`
465  List of data references to Warps to be coadded.
466  Note: `warpRefList` is just the new name for `tempExpRefList`.
467 
468  Returns
469  -------
470  retStruct : `lsst.pipe.base.Struct`
471  Result struct with components:
472 
473  - ``coaddExposure``: coadded exposure (``Exposure``).
474  - ``nImage``: exposure count image (``Image``).
475  """
476  if selectDataList and warpRefList:
477  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
478  "and which to use is ambiguous. Please pass only one.")
479 
480  skyInfo = self.getSkyInfo(dataRef)
481  if warpRefList is None:
482  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
483  if len(calExpRefList) == 0:
484  self.log.warn("No exposures to coadd")
485  return
486  self.log.info("Coadding %d exposures", len(calExpRefList))
487 
488  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
489 
490  inputData = self.prepareInputs(warpRefList)
491  self.log.info("Found %d %s", len(inputData.tempExpRefList),
492  self.getTempExpDatasetName(self.warpType))
493  if len(inputData.tempExpRefList) == 0:
494  self.log.warn("No coadd temporary exposures found")
495  return
496 
497  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
498 
499  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
500  inputData.weightList, supplementaryData=supplementaryData)
501 
502  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
503  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
504 
505  if self.config.doWrite:
506  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
507  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
508  else:
509  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
510  self.log.info("Persisting %s" % coaddDatasetName)
511  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
512  if self.config.doNImage and retStruct.nImage is not None:
513  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
514 
515  return retStruct
516 
517  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
518  """Interpolate over missing data and mask bright stars.
519 
520  Parameters
521  ----------
522  coaddExposure : `lsst.afw.image.Exposure`
523  The coadded exposure to process.
524  dataRef : `lsst.daf.persistence.ButlerDataRef`
525  Butler data reference for supplementary data.
526  """
527  if self.config.doInterp:
528  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
529  # The variance must be positive; work around for DM-3201.
530  varArray = coaddExposure.variance.array
531  with numpy.errstate(invalid="ignore"):
532  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
533 
534  if self.config.doMaskBrightObjects:
535  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
536 
537  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
538  """Make additional inputs to run() specific to subclasses (Gen2)
539 
540  Duplicates interface of `runDataRef` method
541  Available to be implemented by subclasses only if they need the
542  coadd dataRef for performing preliminary processing before
543  assembling the coadd.
544 
545  Parameters
546  ----------
547  dataRef : `lsst.daf.persistence.ButlerDataRef`
548  Butler data reference for supplementary data.
549  selectDataList : `list` (optional)
550  Optional List of data references to Calexps.
551  warpRefList : `list` (optional)
552  Optional List of data references to Warps.
553  """
554  return pipeBase.Struct()
555 
556  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
557  """Make additional inputs to run() specific to subclasses (Gen3)
558 
559  Duplicates interface of `runQuantum` method.
560  Available to be implemented by subclasses only if they need the
561  coadd dataRef for performing preliminary processing before
562  assembling the coadd.
563 
564  Parameters
565  ----------
566  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
567  Gen3 Butler object for fetching additional data products before
568  running the Task specialized for quantum being processed
569  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
570  Attributes are the names of the connections describing input dataset types.
571  Values are DatasetRefs that task consumes for corresponding dataset type.
572  DataIds are guaranteed to match data objects in ``inputData``.
573  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
574  Attributes are the names of the connections describing output dataset types.
575  Values are DatasetRefs that task is to produce
576  for corresponding dataset type.
577  """
578  return pipeBase.Struct()
579 
580  def getTempExpRefList(self, patchRef, calExpRefList):
581  """Generate list data references corresponding to warped exposures
582  that lie within the patch to be coadded.
583 
584  Parameters
585  ----------
586  patchRef : `dataRef`
587  Data reference for patch.
588  calExpRefList : `list`
589  List of data references for input calexps.
590 
591  Returns
592  -------
593  tempExpRefList : `list`
594  List of Warp/CoaddTempExp data references.
595  """
596  butler = patchRef.getButler()
597  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
598  self.getTempExpDatasetName(self.warpType))
599  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
600  g, groupData.keys) for
601  g in groupData.groups.keys()]
602  return tempExpRefList
603 
604  def prepareInputs(self, refList):
605  """Prepare the input warps for coaddition by measuring the weight for
606  each warp and the scaling for the photometric zero point.
607 
608  Each Warp has its own photometric zeropoint and background variance.
609  Before coadding these Warps together, compute a scale factor to
610  normalize the photometric zeropoint and compute the weight for each Warp.
611 
612  Parameters
613  ----------
614  refList : `list`
615  List of data references to tempExp
616 
617  Returns
618  -------
619  result : `lsst.pipe.base.Struct`
620  Result struct with components:
621 
622  - ``tempExprefList``: `list` of data references to tempExp.
623  - ``weightList``: `list` of weightings.
624  - ``imageScalerList``: `list` of image scalers.
625  """
626  statsCtrl = afwMath.StatisticsControl()
627  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
628  statsCtrl.setNumIter(self.config.clipIter)
629  statsCtrl.setAndMask(self.getBadPixelMask())
630  statsCtrl.setNanSafe(True)
631  # compute tempExpRefList: a list of tempExpRef that actually exist
632  # and weightList: a list of the weight of the associated coadd tempExp
633  # and imageScalerList: a list of scale factors for the associated coadd tempExp
634  tempExpRefList = []
635  weightList = []
636  imageScalerList = []
637  tempExpName = self.getTempExpDatasetName(self.warpType)
638  for tempExpRef in refList:
639  # Gen3's DeferredDatasetHandles are guaranteed to exist and
640  # therefore have no datasetExists() method
641  if not isinstance(tempExpRef, DeferredDatasetHandle):
642  if not tempExpRef.datasetExists(tempExpName):
643  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
644  continue
645 
646  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
647  # Ignore any input warp that is empty of data
648  if numpy.isnan(tempExp.image.array).all():
649  continue
650  maskedImage = tempExp.getMaskedImage()
651  imageScaler = self.scaleZeroPoint.computeImageScaler(
652  exposure=tempExp,
653  dataRef=tempExpRef,
654  )
655  try:
656  imageScaler.scaleMaskedImage(maskedImage)
657  except Exception as e:
658  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
659  continue
660  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
661  afwMath.MEANCLIP, statsCtrl)
662  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
663  weight = 1.0 / float(meanVar)
664  if not numpy.isfinite(weight):
665  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
666  continue
667  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
668 
669  del maskedImage
670  del tempExp
671 
672  tempExpRefList.append(tempExpRef)
673  weightList.append(weight)
674  imageScalerList.append(imageScaler)
675 
676  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
677  imageScalerList=imageScalerList)
678 
679  def prepareStats(self, mask=None):
680  """Prepare the statistics for coadding images.
681 
682  Parameters
683  ----------
684  mask : `int`, optional
685  Bit mask value to exclude from coaddition.
686 
687  Returns
688  -------
689  stats : `lsst.pipe.base.Struct`
690  Statistics structure with the following fields:
691 
692  - ``statsCtrl``: Statistics control object for coadd
693  (`lsst.afw.math.StatisticsControl`)
694  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
695  """
696  if mask is None:
697  mask = self.getBadPixelMask()
698  statsCtrl = afwMath.StatisticsControl()
699  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
700  statsCtrl.setNumIter(self.config.clipIter)
701  statsCtrl.setAndMask(mask)
702  statsCtrl.setNanSafe(True)
703  statsCtrl.setWeighted(True)
704  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
705  for plane, threshold in self.config.maskPropagationThresholds.items():
706  bit = afwImage.Mask.getMaskPlane(plane)
707  statsCtrl.setMaskPropagationThreshold(bit, threshold)
708  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
709  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
710 
711  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
712  altMaskList=None, mask=None, supplementaryData=None):
713  """Assemble a coadd from input warps
714 
715  Assemble the coadd using the provided list of coaddTempExps. Since
716  the full coadd covers a patch (a large area), the assembly is
717  performed over small areas on the image at a time in order to
718  conserve memory usage. Iterate over subregions within the outer
719  bbox of the patch using `assembleSubregion` to stack the corresponding
720  subregions from the coaddTempExps with the statistic specified.
721  Set the edge bits the coadd mask based on the weight map.
722 
723  Parameters
724  ----------
725  skyInfo : `lsst.pipe.base.Struct`
726  Struct with geometric information about the patch.
727  tempExpRefList : `list`
728  List of data references to Warps (previously called CoaddTempExps).
729  imageScalerList : `list`
730  List of image scalers.
731  weightList : `list`
732  List of weights
733  altMaskList : `list`, optional
734  List of alternate masks to use rather than those stored with
735  tempExp.
736  mask : `int`, optional
737  Bit mask value to exclude from coaddition.
738  supplementaryData : lsst.pipe.base.Struct, optional
739  Struct with additional data products needed to assemble coadd.
740  Only used by subclasses that implement `makeSupplementaryData`
741  and override `run`.
742 
743  Returns
744  -------
745  result : `lsst.pipe.base.Struct`
746  Result struct with components:
747 
748  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
749  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
750  - ``warpRefList``: input list of refs to the warps (
751  ``lsst.daf.butler.DeferredDatasetHandle`` or
752  ``lsst.daf.persistence.ButlerDataRef``)
753  (unmodified)
754  - ``imageScalerList``: input list of image scalers (unmodified)
755  - ``weightList``: input list of weights (unmodified)
756  """
757  tempExpName = self.getTempExpDatasetName(self.warpType)
758  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
759  stats = self.prepareStats(mask=mask)
760 
761  if altMaskList is None:
762  altMaskList = [None]*len(tempExpRefList)
763 
764  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
765  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
766  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
767  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
768  coaddMaskedImage = coaddExposure.getMaskedImage()
769  subregionSizeArr = self.config.subregionSize
770  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
771  # if nImage is requested, create a zero one which can be passed to assembleSubregion
772  if self.config.doNImage:
773  nImage = afwImage.ImageU(skyInfo.bbox)
774  else:
775  nImage = None
776  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
777  try:
778  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
779  weightList, altMaskList, stats.flags, stats.ctrl,
780  nImage=nImage)
781  except Exception as e:
782  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
783 
784  self.setInexactPsf(coaddMaskedImage.getMask())
785  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
786  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
787  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
788  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
789  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
790  weightList=weightList)
791 
792  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
793  """Set the metadata for the coadd.
794 
795  This basic implementation sets the filter from the first input.
796 
797  Parameters
798  ----------
799  coaddExposure : `lsst.afw.image.Exposure`
800  The target exposure for the coadd.
801  tempExpRefList : `list`
802  List of data references to tempExp.
803  weightList : `list`
804  List of weights.
805  """
806  assert len(tempExpRefList) == len(weightList), "Length mismatch"
807  tempExpName = self.getTempExpDatasetName(self.warpType)
808  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
809  # (and we need more than just the PropertySet that contains the header), which is not possible
810  # with the current butler (see #2777).
811  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
812 
813  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
814  # Gen 3 API
815  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
816  else:
817  # Gen 2 API. Delete this when Gen 2 retired
818  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
819  for tempExpRef in tempExpRefList]
820  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
821 
822  coaddExposure.setFilter(tempExpList[0].getFilter())
823  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
824  coaddInputs.ccds.reserve(numCcds)
825  coaddInputs.visits.reserve(len(tempExpList))
826 
827  for tempExp, weight in zip(tempExpList, weightList):
828  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
829 
830  if self.config.doUsePsfMatchedPolygons:
831  self.shrinkValidPolygons(coaddInputs)
832 
833  coaddInputs.visits.sort()
834  if self.warpType == "psfMatched":
835  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
836  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
837  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
838  # having the maximum width (sufficient because square)
839  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
840  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
841  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
842  else:
843  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
844  self.config.coaddPsf.makeControl())
845  coaddExposure.setPsf(psf)
846  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
847  coaddExposure.getWcs())
848  coaddExposure.getInfo().setApCorrMap(apCorrMap)
849  if self.config.doAttachTransmissionCurve:
850  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
851  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
852 
853  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
854  altMaskList, statsFlags, statsCtrl, nImage=None):
855  """Assemble the coadd for a sub-region.
856 
857  For each coaddTempExp, check for (and swap in) an alternative mask
858  if one is passed. Remove mask planes listed in
859  `config.removeMaskPlanes`. Finally, stack the actual exposures using
860  `lsst.afw.math.statisticsStack` with the statistic specified by
861  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
862  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
863  an N-sigma clipped mean where N and iterations are specified by
864  statsCtrl. Assign the stacked subregion back to the coadd.
865 
866  Parameters
867  ----------
868  coaddExposure : `lsst.afw.image.Exposure`
869  The target exposure for the coadd.
870  bbox : `lsst.geom.Box`
871  Sub-region to coadd.
872  tempExpRefList : `list`
873  List of data reference to tempExp.
874  imageScalerList : `list`
875  List of image scalers.
876  weightList : `list`
877  List of weights.
878  altMaskList : `list`
879  List of alternate masks to use rather than those stored with
880  tempExp, or None. Each element is dict with keys = mask plane
881  name to which to add the spans.
882  statsFlags : `lsst.afw.math.Property`
883  Property object for statistic for coadd.
884  statsCtrl : `lsst.afw.math.StatisticsControl`
885  Statistics control object for coadd.
886  nImage : `lsst.afw.image.ImageU`, optional
887  Keeps track of exposure count for each pixel.
888  """
889  self.log.debug("Computing coadd over %s", bbox)
890  tempExpName = self.getTempExpDatasetName(self.warpType)
891  coaddExposure.mask.addMaskPlane("REJECTED")
892  coaddExposure.mask.addMaskPlane("CLIPPED")
893  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
894  maskMap = self.setRejectedMaskMapping(statsCtrl)
895  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
896  maskedImageList = []
897  if nImage is not None:
898  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
899  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
900 
901  if isinstance(tempExpRef, DeferredDatasetHandle):
902  # Gen 3 API
903  exposure = tempExpRef.get(parameters={'bbox': bbox})
904  else:
905  # Gen 2 API. Delete this when Gen 2 retired
906  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
907 
908  maskedImage = exposure.getMaskedImage()
909  mask = maskedImage.getMask()
910  if altMask is not None:
911  self.applyAltMaskPlanes(mask, altMask)
912  imageScaler.scaleMaskedImage(maskedImage)
913 
914  # Add 1 for each pixel which is not excluded by the exclude mask.
915  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
916  if nImage is not None:
917  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
918  if self.config.removeMaskPlanes:
919  self.removeMaskPlanes(maskedImage)
920  maskedImageList.append(maskedImage)
921 
922  with self.timer("stack"):
923  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
924  clipped, # also set output to CLIPPED if sigma-clipped
925  maskMap)
926  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
927  if nImage is not None:
928  nImage.assign(subNImage, bbox)
929 
930  def removeMaskPlanes(self, maskedImage):
931  """Unset the mask of an image for mask planes specified in the config.
932 
933  Parameters
934  ----------
935  maskedImage : `lsst.afw.image.MaskedImage`
936  The masked image to be modified.
937  """
938  mask = maskedImage.getMask()
939  for maskPlane in self.config.removeMaskPlanes:
940  try:
941  mask &= ~mask.getPlaneBitMask(maskPlane)
942  except pexExceptions.InvalidParameterError:
943  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
944  maskPlane)
945 
946  @staticmethod
947  def setRejectedMaskMapping(statsCtrl):
948  """Map certain mask planes of the warps to new planes for the coadd.
949 
950  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
951  or CLIPPED, set it to REJECTED on the coadd.
952  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
953  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
954 
955  Parameters
956  ----------
957  statsCtrl : `lsst.afw.math.StatisticsControl`
958  Statistics control object for coadd
959 
960  Returns
961  -------
962  maskMap : `list` of `tuple` of `int`
963  A list of mappings of mask planes of the warped exposures to
964  mask planes of the coadd.
965  """
966  edge = afwImage.Mask.getPlaneBitMask("EDGE")
967  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
968  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
969  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
970  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
971  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
972  (clipped, clipped)]
973  return maskMap
974 
975  def applyAltMaskPlanes(self, mask, altMaskSpans):
976  """Apply in place alt mask formatted as SpanSets to a mask.
977 
978  Parameters
979  ----------
980  mask : `lsst.afw.image.Mask`
981  Original mask.
982  altMaskSpans : `dict`
983  SpanSet lists to apply. Each element contains the new mask
984  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
985  and list of SpanSets to apply to the mask.
986 
987  Returns
988  -------
989  mask : `lsst.afw.image.Mask`
990  Updated mask.
991  """
992  if self.config.doUsePsfMatchedPolygons:
993  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
994  # Clear away any other masks outside the validPolygons. These pixels are no longer
995  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
996  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
997  # This mask-clearing step must only occur *before* applying the new masks below
998  for spanSet in altMaskSpans['NO_DATA']:
999  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1000 
1001  for plane, spanSetList in altMaskSpans.items():
1002  maskClipValue = mask.addMaskPlane(plane)
1003  for spanSet in spanSetList:
1004  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1005  return mask
1006 
1007  def shrinkValidPolygons(self, coaddInputs):
1008  """Shrink coaddInputs' ccds' ValidPolygons in place.
1009 
1010  Either modify each ccd's validPolygon in place, or if CoaddInputs
1011  does not have a validPolygon, create one from its bbox.
1012 
1013  Parameters
1014  ----------
1015  coaddInputs : `lsst.afw.image.coaddInputs`
1016  Original mask.
1017 
1018  """
1019  for ccd in coaddInputs.ccds:
1020  polyOrig = ccd.getValidPolygon()
1021  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1022  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1023  if polyOrig:
1024  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1025  else:
1026  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1027  ccd.setValidPolygon(validPolygon)
1028 
1029  def readBrightObjectMasks(self, dataRef):
1030  """Retrieve the bright object masks.
1031 
1032  Returns None on failure.
1033 
1034  Parameters
1035  ----------
1036  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1037  A Butler dataRef.
1038 
1039  Returns
1040  -------
1041  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1042  Bright object mask from the Butler object, or None if it cannot
1043  be retrieved.
1044  """
1045  try:
1046  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1047  except Exception as e:
1048  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1049  return None
1050 
1051  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1052  """Set the bright object masks.
1053 
1054  Parameters
1055  ----------
1056  exposure : `lsst.afw.image.Exposure`
1057  Exposure under consideration.
1058  dataId : `lsst.daf.persistence.dataId`
1059  Data identifier dict for patch.
1060  brightObjectMasks : `lsst.afw.table`
1061  Table of bright objects to mask.
1062  """
1063 
1064  if brightObjectMasks is None:
1065  self.log.warn("Unable to apply bright object mask: none supplied")
1066  return
1067  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1068  mask = exposure.getMaskedImage().getMask()
1069  wcs = exposure.getWcs()
1070  plateScale = wcs.getPixelScale().asArcseconds()
1071 
1072  for rec in brightObjectMasks:
1073  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1074  if rec["type"] == "box":
1075  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1076  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1077  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1078 
1079  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1080  bbox = geom.Box2I(center - halfSize, center + halfSize)
1081 
1082  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1083  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1084  spans = afwGeom.SpanSet(bbox)
1085  elif rec["type"] == "circle":
1086  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1087  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1088  else:
1089  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1090  continue
1091  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1092 
1093  def setInexactPsf(self, mask):
1094  """Set INEXACT_PSF mask plane.
1095 
1096  If any of the input images isn't represented in the coadd (due to
1097  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1098  these pixels.
1099 
1100  Parameters
1101  ----------
1102  mask : `lsst.afw.image.Mask`
1103  Coadded exposure's mask, modified in-place.
1104  """
1105  mask.addMaskPlane("INEXACT_PSF")
1106  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1107  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1108  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1109  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1110  array = mask.getArray()
1111  selected = array & (sensorEdge | clipped | rejected) > 0
1112  array[selected] |= inexactPsf
1113 
1114  @classmethod
1115  def _makeArgumentParser(cls):
1116  """Create an argument parser.
1117  """
1118  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1119  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_" +
1120  cls.ConfigClass().warpType + "Warp",
1121  help="data ID, e.g. --id tract=12345 patch=1,2",
1122  ContainerClass=AssembleCoaddDataIdContainer)
1123  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1124  ContainerClass=SelectDataIdContainer)
1125  return parser
1126 
1127  @staticmethod
1128  def _subBBoxIter(bbox, subregionSize):
1129  """Iterate over subregions of a bbox.
1130 
1131  Parameters
1132  ----------
1133  bbox : `lsst.geom.Box2I`
1134  Bounding box over which to iterate.
1135  subregionSize: `lsst.geom.Extent2I`
1136  Size of sub-bboxes.
1137 
1138  Yields
1139  ------
1140  subBBox : `lsst.geom.Box2I`
1141  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1142  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1143  the edges of ``bbox``, but it will never be empty.
1144  """
1145  if bbox.isEmpty():
1146  raise RuntimeError("bbox %s is empty" % (bbox,))
1147  if subregionSize[0] < 1 or subregionSize[1] < 1:
1148  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1149 
1150  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1151  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1152  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1153  subBBox.clip(bbox)
1154  if subBBox.isEmpty():
1155  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1156  "colShift=%s, rowShift=%s" %
1157  (bbox, subregionSize, colShift, rowShift))
1158  yield subBBox
1159 
1160 
1161 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1162  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1163  """
1164 
1165  def makeDataRefList(self, namespace):
1166  """Make self.refList from self.idList.
1167 
1168  Parameters
1169  ----------
1170  namespace
1171  Results of parsing command-line (with ``butler`` and ``log`` elements).
1172  """
1173  datasetType = namespace.config.coaddName + "Coadd"
1174  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1175 
1176  for dataId in self.idList:
1177  # tract and patch are required
1178  for key in keysCoadd:
1179  if key not in dataId:
1180  raise RuntimeError("--id must include " + key)
1181 
1182  dataRef = namespace.butler.dataRef(
1183  datasetType=datasetType,
1184  dataId=dataId,
1185  )
1186  self.refList.append(dataRef)
1187 
1188 
1189 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1190  """Function to count the number of pixels with a specific mask in a
1191  footprint.
1192 
1193  Find the intersection of mask & footprint. Count all pixels in the mask
1194  that are in the intersection that have bitmask set but do not have
1195  ignoreMask set. Return the count.
1196 
1197  Parameters
1198  ----------
1199  mask : `lsst.afw.image.Mask`
1200  Mask to define intersection region by.
1201  footprint : `lsst.afw.detection.Footprint`
1202  Footprint to define the intersection region by.
1203  bitmask
1204  Specific mask that we wish to count the number of occurances of.
1205  ignoreMask
1206  Pixels to not consider.
1207 
1208  Returns
1209  -------
1210  result : `int`
1211  Count of number of pixels in footprint with specified mask.
1212  """
1213  bbox = footprint.getBBox()
1214  bbox.clip(mask.getBBox(afwImage.PARENT))
1215  fp = afwImage.Mask(bbox)
1216  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1217  footprint.spans.setMask(fp, bitmask)
1218  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1219  (subMask.getArray() & ignoreMask) == 0).sum()
1220 
1221 
1222 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1223  """Configuration parameters for the SafeClipAssembleCoaddTask.
1224  """
1225  clipDetection = pexConfig.ConfigurableField(
1226  target=SourceDetectionTask,
1227  doc="Detect sources on difference between unclipped and clipped coadd")
1228  minClipFootOverlap = pexConfig.Field(
1229  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1230  dtype=float,
1231  default=0.6
1232  )
1233  minClipFootOverlapSingle = pexConfig.Field(
1234  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1235  "clipped when only one visit overlaps",
1236  dtype=float,
1237  default=0.5
1238  )
1239  minClipFootOverlapDouble = pexConfig.Field(
1240  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1241  "clipped when two visits overlap",
1242  dtype=float,
1243  default=0.45
1244  )
1245  maxClipFootOverlapDouble = pexConfig.Field(
1246  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1247  "considering two visits",
1248  dtype=float,
1249  default=0.15
1250  )
1251  minBigOverlap = pexConfig.Field(
1252  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1253  "when labeling clipped footprints",
1254  dtype=int,
1255  default=100
1256  )
1257 
1258  def setDefaults(self):
1259  """Set default values for clipDetection.
1260 
1261  Notes
1262  -----
1263  The numeric values for these configuration parameters were
1264  empirically determined, future work may further refine them.
1265  """
1266  AssembleCoaddConfig.setDefaults(self)
1267  self.clipDetection.doTempLocalBackground = False
1268  self.clipDetection.reEstimateBackground = False
1269  self.clipDetection.returnOriginalFootprints = False
1270  self.clipDetection.thresholdPolarity = "both"
1271  self.clipDetection.thresholdValue = 2
1272  self.clipDetection.nSigmaToGrow = 2
1273  self.clipDetection.minPixels = 4
1274  self.clipDetection.isotropicGrow = True
1275  self.clipDetection.thresholdType = "pixel_stdev"
1276  self.sigmaClip = 1.5
1277  self.clipIter = 3
1278  self.statistic = "MEAN"
1279 
1280  def validate(self):
1281  if self.doSigmaClip:
1282  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1283  "Ignoring doSigmaClip.")
1284  self.doSigmaClip = False
1285  if self.statistic != "MEAN":
1286  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1287  "(%s chosen). Please set statistic to MEAN."
1288  % (self.statistic))
1289  AssembleCoaddTask.ConfigClass.validate(self)
1290 
1291 
1292 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1293  """Assemble a coadded image from a set of coadded temporary exposures,
1294  being careful to clip & flag areas with potential artifacts.
1295 
1296  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1297  we clip outliers). The problem with doing this is that when computing the
1298  coadd PSF at a given location, individual visit PSFs from visits with
1299  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1300  In this task, we correct for this behavior by creating a new
1301  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1302  coaddTempExps and the final coadd where
1303 
1304  i. difference imaging suggests that there is an outlier and
1305  ii. this outlier appears on only one or two images.
1306 
1307  Such regions will not contribute to the final coadd. Furthermore, any
1308  routine to determine the coadd PSF can now be cognizant of clipped regions.
1309  Note that the algorithm implemented by this task is preliminary and works
1310  correctly for HSC data. Parameter modifications and or considerable
1311  redesigning of the algorithm is likley required for other surveys.
1312 
1313  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1314  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1315  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1316  if you wish.
1317 
1318  Notes
1319  -----
1320  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1321  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1322  see `baseDebug` for more about ``debug.py`` files.
1323  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1324  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1325  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1326  for further information.
1327 
1328  Examples
1329  --------
1330  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1331  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1332  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1333 
1334  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1335  and filter to be coadded (specified using
1336  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1337  along with a list of coaddTempExps to attempt to coadd (specified using
1338  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1339  Only the coaddTempExps that cover the specified tract and patch will be
1340  coadded. A list of the available optional arguments can be obtained by
1341  calling assembleCoadd.py with the --help command line argument:
1342 
1343  .. code-block:: none
1344 
1345  assembleCoadd.py --help
1346 
1347  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1348  context of multi-band processing, we will generate the HSC-I & -R band
1349  coadds from HSC engineering test data provided in the ci_hsc package.
1350  To begin, assuming that the lsst stack has been already set up, we must
1351  set up the obs_subaru and ci_hsc packages. This defines the environment
1352  variable $CI_HSC_DIR and points at the location of the package. The raw
1353  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1354  the coadds, we must first
1355 
1356  - ``processCcd``
1357  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1358  - ``makeSkyMap``
1359  create a skymap that covers the area of the sky present in the raw exposures
1360  - ``makeCoaddTempExp``
1361  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1362 
1363  We can perform all of these steps by running
1364 
1365  .. code-block:: none
1366 
1367  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1368 
1369  This will produce warped coaddTempExps for each visit. To coadd the
1370  warped data, we call ``assembleCoadd.py`` as follows:
1371 
1372  .. code-block:: none
1373 
1374  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1375  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1376  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1377  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1378  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1379  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1380  --selectId visit=903988 ccd=24
1381 
1382  This will process the HSC-I band data. The results are written in
1383  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1384 
1385  You may also choose to run:
1386 
1387  .. code-block:: none
1388 
1389  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1390  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1391  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1392  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1393  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1394  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1395  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1396  --selectId visit=903346 ccd=12
1397 
1398  to generate the coadd for the HSC-R band if you are interested in following
1399  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1400  """
1401  ConfigClass = SafeClipAssembleCoaddConfig
1402  _DefaultName = "safeClipAssembleCoadd"
1403 
1404  def __init__(self, *args, **kwargs):
1405  AssembleCoaddTask.__init__(self, *args, **kwargs)
1406  schema = afwTable.SourceTable.makeMinimalSchema()
1407  self.makeSubtask("clipDetection", schema=schema)
1408 
1409  @utils.inheritDoc(AssembleCoaddTask)
1410  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1411  """Assemble the coadd for a region.
1412 
1413  Compute the difference of coadds created with and without outlier
1414  rejection to identify coadd pixels that have outlier values in some
1415  individual visits.
1416  Detect clipped regions on the difference image and mark these regions
1417  on the one or two individual coaddTempExps where they occur if there
1418  is significant overlap between the clipped region and a source. This
1419  leaves us with a set of footprints from the difference image that have
1420  been identified as having occured on just one or two individual visits.
1421  However, these footprints were generated from a difference image. It
1422  is conceivable for a large diffuse source to have become broken up
1423  into multiple footprints acrosss the coadd difference in this process.
1424  Determine the clipped region from all overlapping footprints from the
1425  detected sources in each visit - these are big footprints.
1426  Combine the small and big clipped footprints and mark them on a new
1427  bad mask plane.
1428  Generate the coadd using `AssembleCoaddTask.run` without outlier
1429  removal. Clipped footprints will no longer make it into the coadd
1430  because they are marked in the new bad mask plane.
1431 
1432  Notes
1433  -----
1434  args and kwargs are passed but ignored in order to match the call
1435  signature expected by the parent task.
1436  """
1437  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1438  mask = exp.getMaskedImage().getMask()
1439  mask.addMaskPlane("CLIPPED")
1440 
1441  result = self.detectClip(exp, tempExpRefList)
1442 
1443  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1444 
1445  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1446  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1447  # Append big footprints from individual Warps to result.clipSpans
1448  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1449  result.detectionFootprints, maskClipValue, maskDetValue,
1450  exp.getBBox())
1451  # Create mask of the current clipped footprints
1452  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1453  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1454 
1455  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1456  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1457  maskClip |= maskClipBig
1458 
1459  # Assemble coadd from base class, but ignoring CLIPPED pixels
1460  badMaskPlanes = self.config.badMaskPlanes[:]
1461  badMaskPlanes.append("CLIPPED")
1462  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1463  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1464  result.clipSpans, mask=badPixelMask)
1465 
1466  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1467  """Return an exposure that contains the difference between unclipped
1468  and clipped coadds.
1469 
1470  Generate a difference image between clipped and unclipped coadds.
1471  Compute the difference image by subtracting an outlier-clipped coadd
1472  from an outlier-unclipped coadd. Return the difference image.
1473 
1474  Parameters
1475  ----------
1476  skyInfo : `lsst.pipe.base.Struct`
1477  Patch geometry information, from getSkyInfo
1478  tempExpRefList : `list`
1479  List of data reference to tempExp
1480  imageScalerList : `list`
1481  List of image scalers
1482  weightList : `list`
1483  List of weights
1484 
1485  Returns
1486  -------
1487  exp : `lsst.afw.image.Exposure`
1488  Difference image of unclipped and clipped coadd wrapped in an Exposure
1489  """
1490  # Clone and upcast self.config because current self.config is frozen
1491  config = AssembleCoaddConfig()
1492  # getattr necessary because subtasks do not survive Config.toDict()
1493  # exclude connections because the class of self.config.connections is not
1494  # the same as AssembleCoaddConfig.connections, and the connections are not
1495  # needed to run this task anyway.
1496  configIntersection = {k: getattr(self.config, k)
1497  for k, v in self.config.toDict().items() if (k in config.keys() and
1498  k != "connections")}
1499  config.update(**configIntersection)
1500 
1501  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1502  config.statistic = 'MEAN'
1503  task = AssembleCoaddTask(config=config)
1504  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1505 
1506  config.statistic = 'MEANCLIP'
1507  task = AssembleCoaddTask(config=config)
1508  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1509 
1510  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1511  coaddDiff -= coaddClip.getMaskedImage()
1512  exp = afwImage.ExposureF(coaddDiff)
1513  exp.setPsf(coaddMean.getPsf())
1514  return exp
1515 
1516  def detectClip(self, exp, tempExpRefList):
1517  """Detect clipped regions on an exposure and set the mask on the
1518  individual tempExp masks.
1519 
1520  Detect footprints in the difference image after smoothing the
1521  difference image with a Gaussian kernal. Identify footprints that
1522  overlap with one or two input ``coaddTempExps`` by comparing the
1523  computed overlap fraction to thresholds set in the config. A different
1524  threshold is applied depending on the number of overlapping visits
1525  (restricted to one or two). If the overlap exceeds the thresholds,
1526  the footprint is considered "CLIPPED" and is marked as such on the
1527  coaddTempExp. Return a struct with the clipped footprints, the indices
1528  of the ``coaddTempExps`` that end up overlapping with the clipped
1529  footprints, and a list of new masks for the ``coaddTempExps``.
1530 
1531  Parameters
1532  ----------
1533  exp : `lsst.afw.image.Exposure`
1534  Exposure to run detection on.
1535  tempExpRefList : `list`
1536  List of data reference to tempExp.
1537 
1538  Returns
1539  -------
1540  result : `lsst.pipe.base.Struct`
1541  Result struct with components:
1542 
1543  - ``clipFootprints``: list of clipped footprints.
1544  - ``clipIndices``: indices for each ``clippedFootprint`` in
1545  ``tempExpRefList``.
1546  - ``clipSpans``: List of dictionaries containing spanSet lists
1547  to clip. Each element contains the new maskplane name
1548  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1549  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1550  compressed into footprints.
1551  """
1552  mask = exp.getMaskedImage().getMask()
1553  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1554  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1555  # Merge positive and negative together footprints together
1556  fpSet.positive.merge(fpSet.negative)
1557  footprints = fpSet.positive
1558  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1559  ignoreMask = self.getBadPixelMask()
1560 
1561  clipFootprints = []
1562  clipIndices = []
1563  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1564 
1565  # for use by detectClipBig
1566  visitDetectionFootprints = []
1567 
1568  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1569  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1570  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1571 
1572  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1573  for i, warpRef in enumerate(tempExpRefList):
1574  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1575  immediate=True).getMaskedImage().getMask()
1576  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1577  afwImage.PARENT, True)
1578  maskVisitDet &= maskDetValue
1579  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1580  visitDetectionFootprints.append(visitFootprints)
1581 
1582  for j, footprint in enumerate(footprints.getFootprints()):
1583  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1584  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1585 
1586  # build a list of clipped spans for each visit
1587  for j, footprint in enumerate(footprints.getFootprints()):
1588  nPixel = footprint.getArea()
1589  overlap = [] # hold the overlap with each visit
1590  indexList = [] # index of visit in global list
1591  for i in range(len(tempExpRefList)):
1592  ignore = ignoreArr[i, j]
1593  overlapDet = overlapDetArr[i, j]
1594  totPixel = nPixel - ignore
1595 
1596  # If we have more bad pixels than detection skip
1597  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1598  continue
1599  overlap.append(overlapDet/float(totPixel))
1600  indexList.append(i)
1601 
1602  overlap = numpy.array(overlap)
1603  if not len(overlap):
1604  continue
1605 
1606  keep = False # Should this footprint be marked as clipped?
1607  keepIndex = [] # Which tempExps does the clipped footprint belong to
1608 
1609  # If footprint only has one overlap use a lower threshold
1610  if len(overlap) == 1:
1611  if overlap[0] > self.config.minClipFootOverlapSingle:
1612  keep = True
1613  keepIndex = [0]
1614  else:
1615  # This is the general case where only visit should be clipped
1616  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1617  if len(clipIndex) == 1:
1618  keep = True
1619  keepIndex = [clipIndex[0]]
1620 
1621  # Test if there are clipped objects that overlap two different visits
1622  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1623  if len(clipIndex) == 2 and len(overlap) > 3:
1624  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1625  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1626  keep = True
1627  keepIndex = clipIndex
1628 
1629  if not keep:
1630  continue
1631 
1632  for index in keepIndex:
1633  globalIndex = indexList[index]
1634  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1635 
1636  clipIndices.append(numpy.array(indexList)[keepIndex])
1637  clipFootprints.append(footprint)
1638 
1639  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1640  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1641 
1642  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1643  maskClipValue, maskDetValue, coaddBBox):
1644  """Return individual warp footprints for large artifacts and append
1645  them to ``clipList`` in place.
1646 
1647  Identify big footprints composed of many sources in the coadd
1648  difference that may have originated in a large diffuse source in the
1649  coadd. We do this by indentifying all clipped footprints that overlap
1650  significantly with each source in all the coaddTempExps.
1651 
1652  Parameters
1653  ----------
1654  clipList : `list`
1655  List of alt mask SpanSets with clipping information. Modified.
1656  clipFootprints : `list`
1657  List of clipped footprints.
1658  clipIndices : `list`
1659  List of which entries in tempExpClipList each footprint belongs to.
1660  maskClipValue
1661  Mask value of clipped pixels.
1662  maskDetValue
1663  Mask value of detected pixels.
1664  coaddBBox : `lsst.geom.Box`
1665  BBox of the coadd and warps.
1666 
1667  Returns
1668  -------
1669  bigFootprintsCoadd : `list`
1670  List of big footprints
1671  """
1672  bigFootprintsCoadd = []
1673  ignoreMask = self.getBadPixelMask()
1674  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1675  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1676  for footprint in visitFootprints.getFootprints():
1677  footprint.spans.setMask(maskVisitDet, maskDetValue)
1678 
1679  # build a mask of clipped footprints that are in this visit
1680  clippedFootprintsVisit = []
1681  for foot, clipIndex in zip(clipFootprints, clipIndices):
1682  if index not in clipIndex:
1683  continue
1684  clippedFootprintsVisit.append(foot)
1685  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1686  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1687 
1688  bigFootprintsVisit = []
1689  for foot in visitFootprints.getFootprints():
1690  if foot.getArea() < self.config.minBigOverlap:
1691  continue
1692  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1693  if nCount > self.config.minBigOverlap:
1694  bigFootprintsVisit.append(foot)
1695  bigFootprintsCoadd.append(foot)
1696 
1697  for footprint in bigFootprintsVisit:
1698  clippedSpans["CLIPPED"].append(footprint.spans)
1699 
1700  return bigFootprintsCoadd
1701 
1702 
1704  psfMatchedWarps = pipeBase.connectionTypes.Input(
1705  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1706  "Only PSF-Matched Warps make sense for image subtraction. "
1707  "Therefore, they must be an additional declared input."),
1708  name="{inputCoaddName}Coadd_psfMatchedWarp",
1709  storageClass="ExposureF",
1710  dimensions=("tract", "patch", "skymap", "visit"),
1711  deferLoad=True,
1712  multiple=True
1713  )
1714  templateCoadd = pipeBase.connectionTypes.Output(
1715  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1716  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1717  name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1718  storageClass="ExposureF",
1719  dimensions=("tract", "patch", "skymap", "abstract_filter"),
1720  )
1721 
1722  def __init__(self, *, config=None):
1723  super().__init__(config=config)
1724  if not config.assembleStaticSkyModel.doWrite:
1725  self.outputs.remove("templateCoadd")
1726  config.validate()
1727 
1728 
1729 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1730  pipelineConnections=CompareWarpAssembleCoaddConnections):
1731  assembleStaticSkyModel = pexConfig.ConfigurableField(
1732  target=AssembleCoaddTask,
1733  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1734  " naive/first-iteration model of the static sky.",
1735  )
1736  detect = pexConfig.ConfigurableField(
1737  target=SourceDetectionTask,
1738  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1739  )
1740  detectTemplate = pexConfig.ConfigurableField(
1741  target=SourceDetectionTask,
1742  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1743  )
1744  maxNumEpochs = pexConfig.Field(
1745  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1746  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1747  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1748  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1749  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1750  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1751  "than transient and not masked.",
1752  dtype=int,
1753  default=2
1754  )
1755  maxFractionEpochsLow = pexConfig.RangeField(
1756  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1757  "Effective maxNumEpochs = "
1758  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1759  dtype=float,
1760  default=0.4,
1761  min=0., max=1.,
1762  )
1763  maxFractionEpochsHigh = pexConfig.RangeField(
1764  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1765  "Effective maxNumEpochs = "
1766  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1767  dtype=float,
1768  default=0.03,
1769  min=0., max=1.,
1770  )
1771  spatialThreshold = pexConfig.RangeField(
1772  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1773  "temporal criteria. If 0, clip all. If 1, clip none.",
1774  dtype=float,
1775  default=0.5,
1776  min=0., max=1.,
1777  inclusiveMin=True, inclusiveMax=True
1778  )
1779  doScaleWarpVariance = pexConfig.Field(
1780  doc="Rescale Warp variance plane using empirical noise?",
1781  dtype=bool,
1782  default=True,
1783  )
1784  scaleWarpVariance = pexConfig.ConfigurableField(
1785  target=ScaleVarianceTask,
1786  doc="Rescale variance on warps",
1787  )
1788  doPreserveContainedBySource = pexConfig.Field(
1789  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1790  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1791  dtype=bool,
1792  default=True,
1793  )
1794  doPrefilterArtifacts = pexConfig.Field(
1795  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1796  "because they will be excluded anyway. This prevents them from contributing "
1797  "to the outlier epoch count image and potentially being labeled as persistant."
1798  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1799  dtype=bool,
1800  default=True
1801  )
1802  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1803  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1804  dtype=str,
1805  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1806  )
1807  prefilterArtifactsRatio = pexConfig.Field(
1808  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1809  dtype=float,
1810  default=0.05
1811  )
1812 
1813  def setDefaults(self):
1814  AssembleCoaddConfig.setDefaults(self)
1815  self.statistic = 'MEAN'
1817 
1818  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1819  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1820  if "EDGE" in self.badMaskPlanes:
1821  self.badMaskPlanes.remove('EDGE')
1822  self.removeMaskPlanes.append('EDGE')
1823  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1824  self.assembleStaticSkyModel.warpType = 'psfMatched'
1825  self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1826  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1827  self.assembleStaticSkyModel.sigmaClip = 2.5
1828  self.assembleStaticSkyModel.clipIter = 3
1829  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1830  self.assembleStaticSkyModel.doWrite = False
1831  self.detect.doTempLocalBackground = False
1832  self.detect.reEstimateBackground = False
1833  self.detect.returnOriginalFootprints = False
1834  self.detect.thresholdPolarity = "both"
1835  self.detect.thresholdValue = 5
1836  self.detect.nSigmaToGrow = 2
1837  self.detect.minPixels = 4
1838  self.detect.isotropicGrow = True
1839  self.detect.thresholdType = "pixel_stdev"
1840  self.detectTemplate.nSigmaToGrow = 2.0
1841  self.detectTemplate.doTempLocalBackground = False
1842  self.detectTemplate.reEstimateBackground = False
1843  self.detectTemplate.returnOriginalFootprints = False
1844 
1845  def validate(self):
1846  super().validate()
1847  if self.assembleStaticSkyModel.doNImage:
1848  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1849  "Please set assembleStaticSkyModel.doNImage=False")
1850 
1851  if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1852  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1853  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1854  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1855  "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1856 
1857 
1858 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1859  """Assemble a compareWarp coadded image from a set of warps
1860  by masking artifacts detected by comparing PSF-matched warps.
1861 
1862  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1863  we clip outliers). The problem with doing this is that when computing the
1864  coadd PSF at a given location, individual visit PSFs from visits with
1865  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1866  In this task, we correct for this behavior by creating a new badMaskPlane
1867  'CLIPPED' which marks pixels in the individual warps suspected to contain
1868  an artifact. We populate this plane on the input warps by comparing
1869  PSF-matched warps with a PSF-matched median coadd which serves as a
1870  model of the static sky. Any group of pixels that deviates from the
1871  PSF-matched template coadd by more than config.detect.threshold sigma,
1872  is an artifact candidate. The candidates are then filtered to remove
1873  variable sources and sources that are difficult to subtract such as
1874  bright stars. This filter is configured using the config parameters
1875  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1876  the maximum fraction of epochs that the deviation can appear in and still
1877  be considered an artifact. The spatialThreshold is the maximum fraction of
1878  pixels in the footprint of the deviation that appear in other epochs
1879  (where other epochs is defined by the temporalThreshold). If the deviant
1880  region meets this criteria of having a significant percentage of pixels
1881  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1882  set in the mask. These regions will not contribute to the final coadd.
1883  Furthermore, any routine to determine the coadd PSF can now be cognizant
1884  of clipped regions. Note that the algorithm implemented by this task is
1885  preliminary and works correctly for HSC data. Parameter modifications and
1886  or considerable redesigning of the algorithm is likley required for other
1887  surveys.
1888 
1889  ``CompareWarpAssembleCoaddTask`` sub-classes
1890  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1891  as a subtask to generate the TemplateCoadd (the model of the static sky).
1892 
1893  Notes
1894  -----
1895  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1896  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1897  ``baseDebug`` for more about ``debug.py`` files.
1898 
1899  This task supports the following debug variables:
1900 
1901  - ``saveCountIm``
1902  If True then save the Epoch Count Image as a fits file in the `figPath`
1903  - ``figPath``
1904  Path to save the debug fits images and figures
1905 
1906  For example, put something like:
1907 
1908  .. code-block:: python
1909 
1910  import lsstDebug
1911  def DebugInfo(name):
1912  di = lsstDebug.getInfo(name)
1913  if name == "lsst.pipe.tasks.assembleCoadd":
1914  di.saveCountIm = True
1915  di.figPath = "/desired/path/to/debugging/output/images"
1916  return di
1917  lsstDebug.Info = DebugInfo
1918 
1919  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1920  ``--debug`` flag. Some subtasks may have their own debug variables;
1921  see individual Task documentation.
1922 
1923  Examples
1924  --------
1925  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1926  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1927  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1928  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1929  and filter to be coadded (specified using
1930  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1931  along with a list of coaddTempExps to attempt to coadd (specified using
1932  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1933  Only the warps that cover the specified tract and patch will be coadded.
1934  A list of the available optional arguments can be obtained by calling
1935  ``assembleCoadd.py`` with the ``--help`` command line argument:
1936 
1937  .. code-block:: none
1938 
1939  assembleCoadd.py --help
1940 
1941  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1942  context of multi-band processing, we will generate the HSC-I & -R band
1943  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1944  To begin, assuming that the lsst stack has been already set up, we must
1945  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1946  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1947  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1948  directory. To begin assembling the coadds, we must first
1949 
1950  - processCcd
1951  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1952  - makeSkyMap
1953  create a skymap that covers the area of the sky present in the raw exposures
1954  - makeCoaddTempExp
1955  warp the individual calibrated exposures to the tangent plane of the coadd
1956 
1957  We can perform all of these steps by running
1958 
1959  .. code-block:: none
1960 
1961  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1962 
1963  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1964  warped data, we call ``assembleCoadd.py`` as follows:
1965 
1966  .. code-block:: none
1967 
1968  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1969  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1970  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1971  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1972  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1973  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1974  --selectId visit=903988 ccd=24
1975 
1976  This will process the HSC-I band data. The results are written in
1977  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1978  """
1979  ConfigClass = CompareWarpAssembleCoaddConfig
1980  _DefaultName = "compareWarpAssembleCoadd"
1981 
1982  def __init__(self, *args, **kwargs):
1983  AssembleCoaddTask.__init__(self, *args, **kwargs)
1984  self.makeSubtask("assembleStaticSkyModel")
1985  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1986  self.makeSubtask("detect", schema=detectionSchema)
1987  if self.config.doPreserveContainedBySource:
1988  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1989  if self.config.doScaleWarpVariance:
1990  self.makeSubtask("scaleWarpVariance")
1991 
1992  @utils.inheritDoc(AssembleCoaddTask)
1993  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
1994  """
1995  Generate a templateCoadd to use as a naive model of static sky to
1996  subtract from PSF-Matched warps.
1997 
1998  Returns
1999  -------
2000  result : `lsst.pipe.base.Struct`
2001  Result struct with components:
2002 
2003  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2004  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2005  """
2006  # Ensure that psfMatchedWarps are used as input warps for template generation
2007  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2008  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2009 
2010  # Because subtasks don't have connections we have to make one.
2011  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2012  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2013  if self.config.assembleStaticSkyModel.doWrite:
2014  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2015  # Remove template coadd from both subtask's and main tasks outputs,
2016  # because it is handled by the subtask as `coaddExposure`
2017  del outputRefs.templateCoadd
2018  del staticSkyModelOutputRefs.templateCoadd
2019 
2020  # A PSF-Matched nImage does not exist as a dataset type
2021  del staticSkyModelOutputRefs.nImage
2022 
2023  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2024  staticSkyModelOutputRefs)
2025  if templateCoadd is None:
2026  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2027 
2028  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2029  nImage=templateCoadd.nImage,
2030  warpRefList=templateCoadd.warpRefList,
2031  imageScalerList=templateCoadd.imageScalerList,
2032  weightList=templateCoadd.weightList)
2033 
2034  @utils.inheritDoc(AssembleCoaddTask)
2035  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2036  """
2037  Generate a templateCoadd to use as a naive model of static sky to
2038  subtract from PSF-Matched warps.
2039 
2040  Returns
2041  -------
2042  result : `lsst.pipe.base.Struct`
2043  Result struct with components:
2044 
2045  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2046  - ``nImage``: N Image (``lsst.afw.image.Image``)
2047  """
2048  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2049  if templateCoadd is None:
2050  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2051 
2052  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2053  nImage=templateCoadd.nImage,
2054  warpRefList=templateCoadd.warpRefList,
2055  imageScalerList=templateCoadd.imageScalerList,
2056  weightList=templateCoadd.weightList)
2057 
2058  def _noTemplateMessage(self, warpType):
2059  warpName = (warpType[0].upper() + warpType[1:])
2060  message = """No %(warpName)s warps were found to build the template coadd which is
2061  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2062  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2063  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2064 
2065  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2066  another algorithm like:
2067 
2068  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2069  config.assemble.retarget(SafeClipAssembleCoaddTask)
2070  """ % {"warpName": warpName}
2071  return message
2072 
2073  @utils.inheritDoc(AssembleCoaddTask)
2074  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2075  supplementaryData, *args, **kwargs):
2076  """Assemble the coadd.
2077 
2078  Find artifacts and apply them to the warps' masks creating a list of
2079  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2080  plane. Then pass these alternative masks to the base class's `run`
2081  method.
2082 
2083  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2084  that must contain a ``templateCoadd`` that serves as the
2085  model of the static sky.
2086  """
2087 
2088  # In Gen3 only, check and match the order of the supplementaryData
2089  # (PSF-matched) inputs to the order of the direct inputs,
2090  # so that the artifact mask is applied to the right warp
2091  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
2092  dataIds = [ref.datasetRefOrType.dataId for ref in tempExpRefList]
2093  psfMatchedDataIds = [ref.datasetRefOrType.dataId for ref in supplementaryData.warpRefList]
2094  if dataIds != psfMatchedDataIds:
2095  self.log.info("Reordering and or/padding PSF-matched visit input list")
2096  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2097  psfMatchedDataIds, dataIds)
2098  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2099  psfMatchedDataIds, dataIds)
2100 
2101  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2102  spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2103  supplementaryData.warpRefList,
2104  supplementaryData.imageScalerList)
2105 
2106  badMaskPlanes = self.config.badMaskPlanes[:]
2107  badMaskPlanes.append("CLIPPED")
2108  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2109 
2110  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2111  spanSetMaskList, mask=badPixelMask)
2112 
2113  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2114  # Psf-Matching moves the real edge inwards
2115  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2116  return result
2117 
2118  def applyAltEdgeMask(self, mask, altMaskList):
2119  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2120 
2121  Parameters
2122  ----------
2123  mask : `lsst.afw.image.Mask`
2124  Original mask.
2125  altMaskList : `list`
2126  List of Dicts containing ``spanSet`` lists.
2127  Each element contains the new mask plane name (e.g. "CLIPPED
2128  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2129  the mask.
2130  """
2131  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2132  for visitMask in altMaskList:
2133  if "EDGE" in visitMask:
2134  for spanSet in visitMask['EDGE']:
2135  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2136 
2137  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2138  """Find artifacts.
2139 
2140  Loop through warps twice. The first loop builds a map with the count
2141  of how many epochs each pixel deviates from the templateCoadd by more
2142  than ``config.chiThreshold`` sigma. The second loop takes each
2143  difference image and filters the artifacts detected in each using
2144  count map to filter out variable sources and sources that are
2145  difficult to subtract cleanly.
2146 
2147  Parameters
2148  ----------
2149  templateCoadd : `lsst.afw.image.Exposure`
2150  Exposure to serve as model of static sky.
2151  tempExpRefList : `list`
2152  List of data references to warps.
2153  imageScalerList : `list`
2154  List of image scalers.
2155 
2156  Returns
2157  -------
2158  altMasks : `list`
2159  List of dicts containing information about CLIPPED
2160  (i.e., artifacts), NO_DATA, and EDGE pixels.
2161  """
2162 
2163  self.log.debug("Generating Count Image, and mask lists.")
2164  coaddBBox = templateCoadd.getBBox()
2165  slateIm = afwImage.ImageU(coaddBBox)
2166  epochCountImage = afwImage.ImageU(coaddBBox)
2167  nImage = afwImage.ImageU(coaddBBox)
2168  spanSetArtifactList = []
2169  spanSetNoDataMaskList = []
2170  spanSetEdgeList = []
2171  badPixelMask = self.getBadPixelMask()
2172 
2173  # mask of the warp diffs should = that of only the warp
2174  templateCoadd.mask.clearAllMaskPlanes()
2175 
2176  if self.config.doPreserveContainedBySource:
2177  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2178  else:
2179  templateFootprints = None
2180 
2181  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2182  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2183  if warpDiffExp is not None:
2184  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2185  nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2186  ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2187  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2188  fpSet.positive.merge(fpSet.negative)
2189  footprints = fpSet.positive
2190  slateIm.set(0)
2191  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2192 
2193  # Remove artifacts due to defects before they contribute to the epochCountImage
2194  if self.config.doPrefilterArtifacts:
2195  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2196  for spans in spanSetList:
2197  spans.setImage(slateIm, 1, doClip=True)
2198  epochCountImage += slateIm
2199 
2200  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2201  # undergo a second convolution. Pixels with data in the direct warp
2202  # but not in the PSF-matched warp will not have their artifacts detected.
2203  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2204  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2205  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2206  nansMask.setXY0(warpDiffExp.getXY0())
2207  edgeMask = warpDiffExp.mask
2208  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2209  edgeMask.getPlaneBitMask("EDGE")).split()
2210  else:
2211  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2212  # In this case, mask the whole epoch
2213  nansMask = afwImage.MaskX(coaddBBox, 1)
2214  spanSetList = []
2215  spanSetEdgeMask = []
2216 
2217  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2218 
2219  spanSetNoDataMaskList.append(spanSetNoDataMask)
2220  spanSetArtifactList.append(spanSetList)
2221  spanSetEdgeList.append(spanSetEdgeMask)
2222 
2223  if lsstDebug.Info(__name__).saveCountIm:
2224  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2225  epochCountImage.writeFits(path)
2226 
2227  for i, spanSetList in enumerate(spanSetArtifactList):
2228  if spanSetList:
2229  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2230  templateFootprints)
2231  spanSetArtifactList[i] = filteredSpanSetList
2232 
2233  altMasks = []
2234  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2235  altMasks.append({'CLIPPED': artifacts,
2236  'NO_DATA': noData,
2237  'EDGE': edge})
2238  return altMasks
2239 
2240  def prefilterArtifacts(self, spanSetList, exp):
2241  """Remove artifact candidates covered by bad mask plane.
2242 
2243  Any future editing of the candidate list that does not depend on
2244  temporal information should go in this method.
2245 
2246  Parameters
2247  ----------
2248  spanSetList : `list`
2249  List of SpanSets representing artifact candidates.
2250  exp : `lsst.afw.image.Exposure`
2251  Exposure containing mask planes used to prefilter.
2252 
2253  Returns
2254  -------
2255  returnSpanSetList : `list`
2256  List of SpanSets with artifacts.
2257  """
2258  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2259  goodArr = (exp.mask.array & badPixelMask) == 0
2260  returnSpanSetList = []
2261  bbox = exp.getBBox()
2262  x0, y0 = exp.getXY0()
2263  for i, span in enumerate(spanSetList):
2264  y, x = span.clippedTo(bbox).indices()
2265  yIndexLocal = numpy.array(y) - y0
2266  xIndexLocal = numpy.array(x) - x0
2267  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2268  if goodRatio > self.config.prefilterArtifactsRatio:
2269  returnSpanSetList.append(span)
2270  return returnSpanSetList
2271 
2272  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2273  """Filter artifact candidates.
2274 
2275  Parameters
2276  ----------
2277  spanSetList : `list`
2278  List of SpanSets representing artifact candidates.
2279  epochCountImage : `lsst.afw.image.Image`
2280  Image of accumulated number of warpDiff detections.
2281  nImage : `lsst.afw.image.Image`
2282  Image of the accumulated number of total epochs contributing.
2283 
2284  Returns
2285  -------
2286  maskSpanSetList : `list`
2287  List of SpanSets with artifacts.
2288  """
2289 
2290  maskSpanSetList = []
2291  x0, y0 = epochCountImage.getXY0()
2292  for i, span in enumerate(spanSetList):
2293  y, x = span.indices()
2294  yIdxLocal = [y1 - y0 for y1 in y]
2295  xIdxLocal = [x1 - x0 for x1 in x]
2296  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2297  totalN = nImage.array[yIdxLocal, xIdxLocal]
2298 
2299  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2300  effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2301  self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2302  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2303  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2304  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2305  (outlierN <= effectiveMaxNumEpochs))
2306  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2307  if percentBelowThreshold > self.config.spatialThreshold:
2308  maskSpanSetList.append(span)
2309 
2310  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2311  # If a candidate is contained by a footprint on the template coadd, do not clip
2312  filteredMaskSpanSetList = []
2313  for span in maskSpanSetList:
2314  doKeep = True
2315  for footprint in footprintsToExclude.positive.getFootprints():
2316  if footprint.spans.contains(span):
2317  doKeep = False
2318  break
2319  if doKeep:
2320  filteredMaskSpanSetList.append(span)
2321  maskSpanSetList = filteredMaskSpanSetList
2322 
2323  return maskSpanSetList
2324 
2325  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2326  """Fetch a warp from the butler and return a warpDiff.
2327 
2328  Parameters
2329  ----------
2330  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2331  Butler dataRef for the warp.
2332  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2333  An image scaler object.
2334  templateCoadd : `lsst.afw.image.Exposure`
2335  Exposure to be substracted from the scaled warp.
2336 
2337  Returns
2338  -------
2339  warp : `lsst.afw.image.Exposure`
2340  Exposure of the image difference between the warp and template.
2341  """
2342 
2343  # If the PSF-Matched warp did not exist for this direct warp
2344  # None is holding its place to maintain order in Gen 3
2345  if warpRef is None:
2346  return None
2347  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2348  warpName = self.getTempExpDatasetName('psfMatched')
2349  if not isinstance(warpRef, DeferredDatasetHandle):
2350  if not warpRef.datasetExists(warpName):
2351  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2352  return None
2353  warp = warpRef.get(datasetType=warpName, immediate=True)
2354  # direct image scaler OK for PSF-matched Warp
2355  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2356  mi = warp.getMaskedImage()
2357  if self.config.doScaleWarpVariance:
2358  try:
2359  self.scaleWarpVariance.run(mi)
2360  except Exception as exc:
2361  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2362  mi -= templateCoadd.getMaskedImage()
2363  return warp
2364 
2365  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2366  """Return a path to which to write debugging output.
2367 
2368  Creates a hyphen-delimited string of dataId values for simple filenames.
2369 
2370  Parameters
2371  ----------
2372  prefix : `str`
2373  Prefix for filename.
2374  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2375  Butler dataRef to make the path from.
2376  coaddLevel : `bool`, optional.
2377  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2378  'filter', but no 'visit').
2379 
2380  Returns
2381  -------
2382  result : `str`
2383  Path for debugging output.
2384  """
2385  if coaddLevel:
2386  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2387  else:
2388  keys = warpRef.dataId.keys()
2389  keyList = sorted(keys, reverse=True)
2390  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2391  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2392  return os.path.join(directory, filename)
2393 
2394 
2395 def reorderAndPadList(inputList, inputKey, outputKey, padWith=None):
2396  """Match the order of one list to another, padding if necessary
2397 
2398  Parameters
2399  ----------
2400  inputList : `list`
2401  List to be reordered and padded
2402  inputKey : Any value that can be compared with outputKey
2403  outputKey : Any value that can be compared with inputKey
2404  padWith : Any value to be inserted where inputKey not in outputKeys
2405 
2406  Returns
2407  -------
2408  outputList : `list`
2409  Copy of inputList reordered and padded with `padWith` to match outputList
2410  """
2411  outputList = []
2412  for d in outputKey:
2413  if d in inputKey:
2414  outputList.append(inputList[inputKey.index(d)])
2415  else:
2416  outputList.append(padWith)
2417  return outputList
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
Base class for coaddition.
Definition: coaddBase.py:101
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareStats(self, mask=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:249
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def readBrightObjectMasks(self, dataRef)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:306
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def prepareInputs(self, refList)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def removeMaskPlanes(self, maskedImage)
def reorderAndPadList(inputList, inputKey, outputKey, padWith=None)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)