lsst.pipe.tasks  20.0.0-20-gbb9d1f89+598b390d6c
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41 from .interpImage import InterpImageTask
42 from .scaleZeroPoint import ScaleZeroPointTask
43 from .coaddHelpers import groupPatchExposures, getGroupDataRef
44 from .scaleVariance import ScaleVarianceTask
45 from lsst.meas.algorithms import SourceDetectionTask
46 from lsst.daf.butler import DeferredDatasetHandle
47 
48 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
49  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
50  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
51 
52 
53 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
54  dimensions=("tract", "patch", "abstract_filter", "skymap"),
55  defaultTemplates={"inputCoaddName": "deep",
56  "outputCoaddName": "deep",
57  "warpType": "direct",
58  "warpTypeSuffix": "",
59  "fakesType": ""}):
60  inputWarps = pipeBase.connectionTypes.Input(
61  doc=("Input list of warps to be assemebled i.e. stacked."
62  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
63  name="{inputCoaddName}Coadd_{warpType}Warp",
64  storageClass="ExposureF",
65  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
66  deferLoad=True,
67  multiple=True
68  )
69  skyMap = pipeBase.connectionTypes.Input(
70  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
71  name="{inputCoaddName}Coadd_skyMap",
72  storageClass="SkyMap",
73  dimensions=("skymap", ),
74  )
75  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
76  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
77  " BRIGHT_OBJECT."),
78  name="brightObjectMask",
79  storageClass="ObjectMaskCatalog",
80  dimensions=("tract", "patch", "skymap", "abstract_filter"),
81  )
82  coaddExposure = pipeBase.connectionTypes.Output(
83  doc="Output coadded exposure, produced by stacking input warps",
84  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
85  storageClass="ExposureF",
86  dimensions=("tract", "patch", "skymap", "abstract_filter"),
87  )
88  nImage = pipeBase.connectionTypes.Output(
89  doc="Output image of number of input images per pixel",
90  name="{outputCoaddName}Coadd_nImage",
91  storageClass="ImageU",
92  dimensions=("tract", "patch", "skymap", "abstract_filter"),
93  )
94 
95  def __init__(self, *, config=None):
96  # Override the connection's name template with config to replicate Gen2 behavior
97  config.connections.warpType = config.warpType
98  config.connections.warpTypeSuffix = makeCoaddSuffix(config.warpType)
99 
100  if config.hasFakes:
101  config.connections.fakesType = "_fakes"
102 
103  super().__init__(config=config)
104 
105  if not config.doMaskBrightObjects:
106  self.prerequisiteInputs.remove("brightObjectMask")
107 
108  if not config.doNImage:
109  self.outputs.remove("nImage")
110 
111 
112 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
113  pipelineConnections=AssembleCoaddConnections):
114  """Configuration parameters for the `AssembleCoaddTask`.
115 
116  Notes
117  -----
118  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
119  only set the bitplane config.brightObjectMaskName. To make this useful you
120  *must* also configure the flags.pixel algorithm, for example by adding
121 
122  .. code-block:: none
123 
124  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
125  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
126 
127  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
128  """
129  warpType = pexConfig.Field(
130  doc="Warp name: one of 'direct' or 'psfMatched'",
131  dtype=str,
132  default="direct",
133  )
134  subregionSize = pexConfig.ListField(
135  dtype=int,
136  doc="Width, height of stack subregion size; "
137  "make small enough that a full stack of images will fit into memory at once.",
138  length=2,
139  default=(2000, 2000),
140  )
141  statistic = pexConfig.Field(
142  dtype=str,
143  doc="Main stacking statistic for aggregating over the epochs.",
144  default="MEANCLIP",
145  )
146  doSigmaClip = pexConfig.Field(
147  dtype=bool,
148  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
149  default=False,
150  )
151  sigmaClip = pexConfig.Field(
152  dtype=float,
153  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
154  default=3.0,
155  )
156  clipIter = pexConfig.Field(
157  dtype=int,
158  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
159  default=2,
160  )
161  calcErrorFromInputVariance = pexConfig.Field(
162  dtype=bool,
163  doc="Calculate coadd variance from input variance by stacking statistic."
164  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
165  default=True,
166  )
167  scaleZeroPoint = pexConfig.ConfigurableField(
168  target=ScaleZeroPointTask,
169  doc="Task to adjust the photometric zero point of the coadd temp exposures",
170  )
171  doInterp = pexConfig.Field(
172  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
173  dtype=bool,
174  default=True,
175  )
176  interpImage = pexConfig.ConfigurableField(
177  target=InterpImageTask,
178  doc="Task to interpolate (and extrapolate) over NaN pixels",
179  )
180  doWrite = pexConfig.Field(
181  doc="Persist coadd?",
182  dtype=bool,
183  default=True,
184  )
185  doNImage = pexConfig.Field(
186  doc="Create image of number of contributing exposures for each pixel",
187  dtype=bool,
188  default=False,
189  )
190  doUsePsfMatchedPolygons = pexConfig.Field(
191  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
192  dtype=bool,
193  default=False,
194  )
195  maskPropagationThresholds = pexConfig.DictField(
196  keytype=str,
197  itemtype=float,
198  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
199  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
200  "would have contributed exceeds this value."),
201  default={"SAT": 0.1},
202  )
203  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
204  doc="Mask planes to remove before coadding")
205  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
206  doc="Set mask and flag bits for bright objects?")
207  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
208  doc="Name of mask bit used for bright objects")
209  coaddPsf = pexConfig.ConfigField(
210  doc="Configuration for CoaddPsf",
211  dtype=measAlg.CoaddPsfConfig,
212  )
213  doAttachTransmissionCurve = pexConfig.Field(
214  dtype=bool, default=False, optional=False,
215  doc=("Attach a piecewise TransmissionCurve for the coadd? "
216  "(requires all input Exposures to have TransmissionCurves).")
217  )
218  hasFakes = pexConfig.Field(
219  dtype=bool,
220  default=False,
221  doc="Should be set to True if fake sources have been inserted into the input data."
222  )
223 
224  def setDefaults(self):
225  super().setDefaults()
226  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
227 
228  def validate(self):
229  super().validate()
230  if self.doPsfMatch:
231  # Backwards compatibility.
232  # Configs do not have loggers
233  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
234  self.warpType = 'psfMatched'
235  if self.doSigmaClip and self.statistic != "MEANCLIP":
236  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
237  self.statistic = "MEANCLIP"
238  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
239  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
240  "compute and set a non-zero coadd variance estimate." % (self.statistic))
241 
242  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
243  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
244  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
245  if str(k) not in unstackableStats]
246  raise ValueError("statistic %s is not allowed. Please choose one of %s."
247  % (self.statistic, stackableStats))
248 
249 
250 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
251  """Assemble a coadded image from a set of warps (coadded temporary exposures).
252 
253  We want to assemble a coadded image from a set of Warps (also called
254  coadded temporary exposures or ``coaddTempExps``).
255  Each input Warp covers a patch on the sky and corresponds to a single
256  run/visit/exposure of the covered patch. We provide the task with a list
257  of Warps (``selectDataList``) from which it selects Warps that cover the
258  specified patch (pointed at by ``dataRef``).
259  Each Warp that goes into a coadd will typically have an independent
260  photometric zero-point. Therefore, we must scale each Warp to set it to
261  a common photometric zeropoint. WarpType may be one of 'direct' or
262  'psfMatched', and the boolean configs `config.makeDirect` and
263  `config.makePsfMatched` set which of the warp types will be coadded.
264  The coadd is computed as a mean with optional outlier rejection.
265  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
266  Finally, Warps can have bad 'NaN' pixels which received no input from the
267  source calExps. We interpolate over these bad (NaN) pixels.
268 
269  `AssembleCoaddTask` uses several sub-tasks. These are
270 
271  - `ScaleZeroPointTask`
272  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
273  - `InterpImageTask`
274  - interpolate across bad pixels (NaN) in the final coadd
275 
276  You can retarget these subtasks if you wish.
277 
278  Notes
279  -----
280  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
281  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
282  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
283  no debug variables of its own. Some of the subtasks may support debug
284  variables. See the documentation for the subtasks for further information.
285 
286  Examples
287  --------
288  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
289  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
290  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
291  inputs: a data reference to the tract patch and filter to be coadded, and
292  a list of Warps to attempt to coadd. These are specified using ``--id`` and
293  ``--selectId``, respectively:
294 
295  .. code-block:: none
296 
297  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
298  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
299 
300  Only the Warps that cover the specified tract and patch will be coadded.
301  A list of the available optional arguments can be obtained by calling
302  ``assembleCoadd.py`` with the ``--help`` command line argument:
303 
304  .. code-block:: none
305 
306  assembleCoadd.py --help
307 
308  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
309  multi-band processing, we will generate the HSC-I & -R band coadds from
310  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
311  assuming that the lsst stack has been already set up, we must set up the
312  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
313  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
314  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
315  coadds, we must first
316 
317  - processCcd
318  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
319  - makeSkyMap
320  - create a skymap that covers the area of the sky present in the raw exposures
321  - makeCoaddTempExp
322  - warp the individual calibrated exposures to the tangent plane of the coadd
323 
324  We can perform all of these steps by running
325 
326  .. code-block:: none
327 
328  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
329 
330  This will produce warped exposures for each visit. To coadd the warped
331  data, we call assembleCoadd.py as follows:
332 
333  .. code-block:: none
334 
335  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
336  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
337  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
338  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
339  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
340  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
341  --selectId visit=903988 ccd=24
342 
343  that will process the HSC-I band data. The results are written in
344  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
345 
346  You may also choose to run:
347 
348  .. code-block:: none
349 
350  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
351  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
352  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
353  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
354  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
355  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
356  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
357  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
358 
359  to generate the coadd for the HSC-R band if you are interested in
360  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
361  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
362  rather than `AssembleCoaddTask` to make the coadd.
363  """
364  ConfigClass = AssembleCoaddConfig
365  _DefaultName = "assembleCoadd"
366 
367  def __init__(self, *args, **kwargs):
368  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
369  if args:
370  argNames = ["config", "name", "parentTask", "log"]
371  kwargs.update({k: v for k, v in zip(argNames, args)})
372  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
373  "PipelineTask will not take positional args" % argNames, FutureWarning)
374 
375  super().__init__(**kwargs)
376  self.makeSubtask("interpImage")
377  self.makeSubtask("scaleZeroPoint")
378 
379  if self.config.doMaskBrightObjects:
380  mask = afwImage.Mask()
381  try:
382  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
383  except pexExceptions.LsstCppException:
384  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
385  mask.getMaskPlaneDict().keys())
386  del mask
387 
388  self.warpType = self.config.warpType
389 
390  @utils.inheritDoc(pipeBase.PipelineTask)
391  def runQuantum(self, butlerQC, inputRefs, outputRefs):
392  # Docstring to be formatted with info from PipelineTask.runQuantum
393  """
394  Notes
395  -----
396  Assemble a coadd from a set of Warps.
397 
398  PipelineTask (Gen3) entry point to Coadd a set of Warps.
399  Analogous to `runDataRef`, it prepares all the data products to be
400  passed to `run`, and processes the results before returning a struct
401  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
402  Therefore, its inputs are accessed subregion by subregion
403  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
404  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
405  correspond to an update in `runDataRef` while both entry points
406  are used.
407  """
408  inputData = butlerQC.get(inputRefs)
409 
410  # Construct skyInfo expected by run
411  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
412  skyMap = inputData["skyMap"]
413  outputDataId = butlerQC.quantum.dataId
414 
415  inputData['skyInfo'] = makeSkyInfo(skyMap,
416  tractId=outputDataId['tract'],
417  patchId=outputDataId['patch'])
418 
419  # Construct list of input Deferred Datasets
420  # These quack a bit like like Gen2 DataRefs
421  warpRefList = inputData['inputWarps']
422  # Perform same middle steps as `runDataRef` does
423  inputs = self.prepareInputs(warpRefList)
424  self.log.info("Found %d %s", len(inputs.tempExpRefList),
425  self.getTempExpDatasetName(self.warpType))
426  if len(inputs.tempExpRefList) == 0:
427  self.log.warn("No coadd temporary exposures found")
428  return
429 
430  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
431  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
432  inputs.weightList, supplementaryData=supplementaryData)
433 
434  inputData.setdefault('brightObjectMask', None)
435  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
436 
437  if self.config.doWrite:
438  butlerQC.put(retStruct, outputRefs)
439  return retStruct
440 
441  @pipeBase.timeMethod
442  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
443  """Assemble a coadd from a set of Warps.
444 
445  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
446  Compute weights to be applied to each Warp and
447  find scalings to match the photometric zeropoint to a reference Warp.
448  Assemble the Warps using `run`. Interpolate over NaNs and
449  optionally write the coadd to disk. Return the coadded exposure.
450 
451  Parameters
452  ----------
453  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
454  Data reference defining the patch for coaddition and the
455  reference Warp (if ``config.autoReference=False``).
456  Used to access the following data products:
457  - ``self.config.coaddName + "Coadd_skyMap"``
458  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
459  - ``self.config.coaddName + "Coadd"``
460  selectDataList : `list`
461  List of data references to Calexps. Data to be coadded will be
462  selected from this list based on overlap with the patch defined
463  by dataRef, grouped by visit, and converted to a list of data
464  references to warps.
465  warpRefList : `list`
466  List of data references to Warps to be coadded.
467  Note: `warpRefList` is just the new name for `tempExpRefList`.
468 
469  Returns
470  -------
471  retStruct : `lsst.pipe.base.Struct`
472  Result struct with components:
473 
474  - ``coaddExposure``: coadded exposure (``Exposure``).
475  - ``nImage``: exposure count image (``Image``).
476  """
477  if selectDataList and warpRefList:
478  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
479  "and which to use is ambiguous. Please pass only one.")
480 
481  skyInfo = self.getSkyInfo(dataRef)
482  if warpRefList is None:
483  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
484  if len(calExpRefList) == 0:
485  self.log.warn("No exposures to coadd")
486  return
487  self.log.info("Coadding %d exposures", len(calExpRefList))
488 
489  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
490 
491  inputData = self.prepareInputs(warpRefList)
492  self.log.info("Found %d %s", len(inputData.tempExpRefList),
493  self.getTempExpDatasetName(self.warpType))
494  if len(inputData.tempExpRefList) == 0:
495  self.log.warn("No coadd temporary exposures found")
496  return
497 
498  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
499 
500  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
501  inputData.weightList, supplementaryData=supplementaryData)
502 
503  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
504  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
505 
506  if self.config.doWrite:
507  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
508  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
509  else:
510  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
511  self.log.info("Persisting %s" % coaddDatasetName)
512  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
513  if self.config.doNImage and retStruct.nImage is not None:
514  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
515 
516  return retStruct
517 
518  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
519  """Interpolate over missing data and mask bright stars.
520 
521  Parameters
522  ----------
523  coaddExposure : `lsst.afw.image.Exposure`
524  The coadded exposure to process.
525  dataRef : `lsst.daf.persistence.ButlerDataRef`
526  Butler data reference for supplementary data.
527  """
528  if self.config.doInterp:
529  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
530  # The variance must be positive; work around for DM-3201.
531  varArray = coaddExposure.variance.array
532  with numpy.errstate(invalid="ignore"):
533  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
534 
535  if self.config.doMaskBrightObjects:
536  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
537 
538  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
539  """Make additional inputs to run() specific to subclasses (Gen2)
540 
541  Duplicates interface of `runDataRef` method
542  Available to be implemented by subclasses only if they need the
543  coadd dataRef for performing preliminary processing before
544  assembling the coadd.
545 
546  Parameters
547  ----------
548  dataRef : `lsst.daf.persistence.ButlerDataRef`
549  Butler data reference for supplementary data.
550  selectDataList : `list` (optional)
551  Optional List of data references to Calexps.
552  warpRefList : `list` (optional)
553  Optional List of data references to Warps.
554  """
555  return pipeBase.Struct()
556 
557  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
558  """Make additional inputs to run() specific to subclasses (Gen3)
559 
560  Duplicates interface of `runQuantum` method.
561  Available to be implemented by subclasses only if they need the
562  coadd dataRef for performing preliminary processing before
563  assembling the coadd.
564 
565  Parameters
566  ----------
567  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
568  Gen3 Butler object for fetching additional data products before
569  running the Task specialized for quantum being processed
570  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
571  Attributes are the names of the connections describing input dataset types.
572  Values are DatasetRefs that task consumes for corresponding dataset type.
573  DataIds are guaranteed to match data objects in ``inputData``.
574  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
575  Attributes are the names of the connections describing output dataset types.
576  Values are DatasetRefs that task is to produce
577  for corresponding dataset type.
578  """
579  return pipeBase.Struct()
580 
581  def getTempExpRefList(self, patchRef, calExpRefList):
582  """Generate list data references corresponding to warped exposures
583  that lie within the patch to be coadded.
584 
585  Parameters
586  ----------
587  patchRef : `dataRef`
588  Data reference for patch.
589  calExpRefList : `list`
590  List of data references for input calexps.
591 
592  Returns
593  -------
594  tempExpRefList : `list`
595  List of Warp/CoaddTempExp data references.
596  """
597  butler = patchRef.getButler()
598  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
599  self.getTempExpDatasetName(self.warpType))
600  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
601  g, groupData.keys) for
602  g in groupData.groups.keys()]
603  return tempExpRefList
604 
605  def prepareInputs(self, refList):
606  """Prepare the input warps for coaddition by measuring the weight for
607  each warp and the scaling for the photometric zero point.
608 
609  Each Warp has its own photometric zeropoint and background variance.
610  Before coadding these Warps together, compute a scale factor to
611  normalize the photometric zeropoint and compute the weight for each Warp.
612 
613  Parameters
614  ----------
615  refList : `list`
616  List of data references to tempExp
617 
618  Returns
619  -------
620  result : `lsst.pipe.base.Struct`
621  Result struct with components:
622 
623  - ``tempExprefList``: `list` of data references to tempExp.
624  - ``weightList``: `list` of weightings.
625  - ``imageScalerList``: `list` of image scalers.
626  """
627  statsCtrl = afwMath.StatisticsControl()
628  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
629  statsCtrl.setNumIter(self.config.clipIter)
630  statsCtrl.setAndMask(self.getBadPixelMask())
631  statsCtrl.setNanSafe(True)
632  # compute tempExpRefList: a list of tempExpRef that actually exist
633  # and weightList: a list of the weight of the associated coadd tempExp
634  # and imageScalerList: a list of scale factors for the associated coadd tempExp
635  tempExpRefList = []
636  weightList = []
637  imageScalerList = []
638  tempExpName = self.getTempExpDatasetName(self.warpType)
639  for tempExpRef in refList:
640  # Gen3's DeferredDatasetHandles are guaranteed to exist and
641  # therefore have no datasetExists() method
642  if not isinstance(tempExpRef, DeferredDatasetHandle):
643  if not tempExpRef.datasetExists(tempExpName):
644  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
645  continue
646 
647  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
648  # Ignore any input warp that is empty of data
649  if numpy.isnan(tempExp.image.array).all():
650  continue
651  maskedImage = tempExp.getMaskedImage()
652  imageScaler = self.scaleZeroPoint.computeImageScaler(
653  exposure=tempExp,
654  dataRef=tempExpRef,
655  )
656  try:
657  imageScaler.scaleMaskedImage(maskedImage)
658  except Exception as e:
659  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
660  continue
661  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
662  afwMath.MEANCLIP, statsCtrl)
663  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
664  weight = 1.0 / float(meanVar)
665  if not numpy.isfinite(weight):
666  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
667  continue
668  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
669 
670  del maskedImage
671  del tempExp
672 
673  tempExpRefList.append(tempExpRef)
674  weightList.append(weight)
675  imageScalerList.append(imageScaler)
676 
677  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
678  imageScalerList=imageScalerList)
679 
680  def prepareStats(self, mask=None):
681  """Prepare the statistics for coadding images.
682 
683  Parameters
684  ----------
685  mask : `int`, optional
686  Bit mask value to exclude from coaddition.
687 
688  Returns
689  -------
690  stats : `lsst.pipe.base.Struct`
691  Statistics structure with the following fields:
692 
693  - ``statsCtrl``: Statistics control object for coadd
694  (`lsst.afw.math.StatisticsControl`)
695  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
696  """
697  if mask is None:
698  mask = self.getBadPixelMask()
699  statsCtrl = afwMath.StatisticsControl()
700  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
701  statsCtrl.setNumIter(self.config.clipIter)
702  statsCtrl.setAndMask(mask)
703  statsCtrl.setNanSafe(True)
704  statsCtrl.setWeighted(True)
705  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
706  for plane, threshold in self.config.maskPropagationThresholds.items():
707  bit = afwImage.Mask.getMaskPlane(plane)
708  statsCtrl.setMaskPropagationThreshold(bit, threshold)
709  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
710  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
711 
712  @pipeBase.timeMethod
713  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
714  altMaskList=None, mask=None, supplementaryData=None):
715  """Assemble a coadd from input warps
716 
717  Assemble the coadd using the provided list of coaddTempExps. Since
718  the full coadd covers a patch (a large area), the assembly is
719  performed over small areas on the image at a time in order to
720  conserve memory usage. Iterate over subregions within the outer
721  bbox of the patch using `assembleSubregion` to stack the corresponding
722  subregions from the coaddTempExps with the statistic specified.
723  Set the edge bits the coadd mask based on the weight map.
724 
725  Parameters
726  ----------
727  skyInfo : `lsst.pipe.base.Struct`
728  Struct with geometric information about the patch.
729  tempExpRefList : `list`
730  List of data references to Warps (previously called CoaddTempExps).
731  imageScalerList : `list`
732  List of image scalers.
733  weightList : `list`
734  List of weights
735  altMaskList : `list`, optional
736  List of alternate masks to use rather than those stored with
737  tempExp.
738  mask : `int`, optional
739  Bit mask value to exclude from coaddition.
740  supplementaryData : lsst.pipe.base.Struct, optional
741  Struct with additional data products needed to assemble coadd.
742  Only used by subclasses that implement `makeSupplementaryData`
743  and override `run`.
744 
745  Returns
746  -------
747  result : `lsst.pipe.base.Struct`
748  Result struct with components:
749 
750  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
751  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
752  - ``warpRefList``: input list of refs to the warps (
753  ``lsst.daf.butler.DeferredDatasetHandle`` or
754  ``lsst.daf.persistence.ButlerDataRef``)
755  (unmodified)
756  - ``imageScalerList``: input list of image scalers (unmodified)
757  - ``weightList``: input list of weights (unmodified)
758  """
759  tempExpName = self.getTempExpDatasetName(self.warpType)
760  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
761  stats = self.prepareStats(mask=mask)
762 
763  if altMaskList is None:
764  altMaskList = [None]*len(tempExpRefList)
765 
766  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
767  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
768  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
769  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
770  coaddMaskedImage = coaddExposure.getMaskedImage()
771  subregionSizeArr = self.config.subregionSize
772  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
773  # if nImage is requested, create a zero one which can be passed to assembleSubregion
774  if self.config.doNImage:
775  nImage = afwImage.ImageU(skyInfo.bbox)
776  else:
777  nImage = None
778  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
779  try:
780  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
781  weightList, altMaskList, stats.flags, stats.ctrl,
782  nImage=nImage)
783  except Exception as e:
784  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
785 
786  self.setInexactPsf(coaddMaskedImage.getMask())
787  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
788  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
789  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
790  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
791  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
792  weightList=weightList)
793 
794  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
795  """Set the metadata for the coadd.
796 
797  This basic implementation sets the filter from the first input.
798 
799  Parameters
800  ----------
801  coaddExposure : `lsst.afw.image.Exposure`
802  The target exposure for the coadd.
803  tempExpRefList : `list`
804  List of data references to tempExp.
805  weightList : `list`
806  List of weights.
807  """
808  assert len(tempExpRefList) == len(weightList), "Length mismatch"
809  tempExpName = self.getTempExpDatasetName(self.warpType)
810  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
811  # (and we need more than just the PropertySet that contains the header), which is not possible
812  # with the current butler (see #2777).
813  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
814 
815  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
816  # Gen 3 API
817  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
818  else:
819  # Gen 2 API. Delete this when Gen 2 retired
820  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
821  for tempExpRef in tempExpRefList]
822  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
823 
824  coaddExposure.setFilter(tempExpList[0].getFilter())
825  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
826  coaddInputs.ccds.reserve(numCcds)
827  coaddInputs.visits.reserve(len(tempExpList))
828 
829  for tempExp, weight in zip(tempExpList, weightList):
830  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
831 
832  if self.config.doUsePsfMatchedPolygons:
833  self.shrinkValidPolygons(coaddInputs)
834 
835  coaddInputs.visits.sort()
836  if self.warpType == "psfMatched":
837  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
838  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
839  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
840  # having the maximum width (sufficient because square)
841  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
842  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
843  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
844  else:
845  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
846  self.config.coaddPsf.makeControl())
847  coaddExposure.setPsf(psf)
848  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
849  coaddExposure.getWcs())
850  coaddExposure.getInfo().setApCorrMap(apCorrMap)
851  if self.config.doAttachTransmissionCurve:
852  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
853  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
854 
855  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
856  altMaskList, statsFlags, statsCtrl, nImage=None):
857  """Assemble the coadd for a sub-region.
858 
859  For each coaddTempExp, check for (and swap in) an alternative mask
860  if one is passed. Remove mask planes listed in
861  `config.removeMaskPlanes`. Finally, stack the actual exposures using
862  `lsst.afw.math.statisticsStack` with the statistic specified by
863  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
864  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
865  an N-sigma clipped mean where N and iterations are specified by
866  statsCtrl. Assign the stacked subregion back to the coadd.
867 
868  Parameters
869  ----------
870  coaddExposure : `lsst.afw.image.Exposure`
871  The target exposure for the coadd.
872  bbox : `lsst.geom.Box`
873  Sub-region to coadd.
874  tempExpRefList : `list`
875  List of data reference to tempExp.
876  imageScalerList : `list`
877  List of image scalers.
878  weightList : `list`
879  List of weights.
880  altMaskList : `list`
881  List of alternate masks to use rather than those stored with
882  tempExp, or None. Each element is dict with keys = mask plane
883  name to which to add the spans.
884  statsFlags : `lsst.afw.math.Property`
885  Property object for statistic for coadd.
886  statsCtrl : `lsst.afw.math.StatisticsControl`
887  Statistics control object for coadd.
888  nImage : `lsst.afw.image.ImageU`, optional
889  Keeps track of exposure count for each pixel.
890  """
891  self.log.debug("Computing coadd over %s", bbox)
892  tempExpName = self.getTempExpDatasetName(self.warpType)
893  coaddExposure.mask.addMaskPlane("REJECTED")
894  coaddExposure.mask.addMaskPlane("CLIPPED")
895  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
896  maskMap = self.setRejectedMaskMapping(statsCtrl)
897  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
898  maskedImageList = []
899  if nImage is not None:
900  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
901  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
902 
903  if isinstance(tempExpRef, DeferredDatasetHandle):
904  # Gen 3 API
905  exposure = tempExpRef.get(parameters={'bbox': bbox})
906  else:
907  # Gen 2 API. Delete this when Gen 2 retired
908  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
909 
910  maskedImage = exposure.getMaskedImage()
911  mask = maskedImage.getMask()
912  if altMask is not None:
913  self.applyAltMaskPlanes(mask, altMask)
914  imageScaler.scaleMaskedImage(maskedImage)
915 
916  # Add 1 for each pixel which is not excluded by the exclude mask.
917  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
918  if nImage is not None:
919  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
920  if self.config.removeMaskPlanes:
921  self.removeMaskPlanes(maskedImage)
922  maskedImageList.append(maskedImage)
923 
924  with self.timer("stack"):
925  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
926  clipped, # also set output to CLIPPED if sigma-clipped
927  maskMap)
928  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
929  if nImage is not None:
930  nImage.assign(subNImage, bbox)
931 
932  def removeMaskPlanes(self, maskedImage):
933  """Unset the mask of an image for mask planes specified in the config.
934 
935  Parameters
936  ----------
937  maskedImage : `lsst.afw.image.MaskedImage`
938  The masked image to be modified.
939  """
940  mask = maskedImage.getMask()
941  for maskPlane in self.config.removeMaskPlanes:
942  try:
943  mask &= ~mask.getPlaneBitMask(maskPlane)
944  except pexExceptions.InvalidParameterError:
945  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
946  maskPlane)
947 
948  @staticmethod
949  def setRejectedMaskMapping(statsCtrl):
950  """Map certain mask planes of the warps to new planes for the coadd.
951 
952  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
953  or CLIPPED, set it to REJECTED on the coadd.
954  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
955  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
956 
957  Parameters
958  ----------
959  statsCtrl : `lsst.afw.math.StatisticsControl`
960  Statistics control object for coadd
961 
962  Returns
963  -------
964  maskMap : `list` of `tuple` of `int`
965  A list of mappings of mask planes of the warped exposures to
966  mask planes of the coadd.
967  """
968  edge = afwImage.Mask.getPlaneBitMask("EDGE")
969  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
970  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
971  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
972  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
973  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
974  (clipped, clipped)]
975  return maskMap
976 
977  def applyAltMaskPlanes(self, mask, altMaskSpans):
978  """Apply in place alt mask formatted as SpanSets to a mask.
979 
980  Parameters
981  ----------
982  mask : `lsst.afw.image.Mask`
983  Original mask.
984  altMaskSpans : `dict`
985  SpanSet lists to apply. Each element contains the new mask
986  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
987  and list of SpanSets to apply to the mask.
988 
989  Returns
990  -------
991  mask : `lsst.afw.image.Mask`
992  Updated mask.
993  """
994  if self.config.doUsePsfMatchedPolygons:
995  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
996  # Clear away any other masks outside the validPolygons. These pixels are no longer
997  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
998  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
999  # This mask-clearing step must only occur *before* applying the new masks below
1000  for spanSet in altMaskSpans['NO_DATA']:
1001  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1002 
1003  for plane, spanSetList in altMaskSpans.items():
1004  maskClipValue = mask.addMaskPlane(plane)
1005  for spanSet in spanSetList:
1006  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1007  return mask
1008 
1009  def shrinkValidPolygons(self, coaddInputs):
1010  """Shrink coaddInputs' ccds' ValidPolygons in place.
1011 
1012  Either modify each ccd's validPolygon in place, or if CoaddInputs
1013  does not have a validPolygon, create one from its bbox.
1014 
1015  Parameters
1016  ----------
1017  coaddInputs : `lsst.afw.image.coaddInputs`
1018  Original mask.
1019 
1020  """
1021  for ccd in coaddInputs.ccds:
1022  polyOrig = ccd.getValidPolygon()
1023  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1024  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1025  if polyOrig:
1026  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1027  else:
1028  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1029  ccd.setValidPolygon(validPolygon)
1030 
1031  def readBrightObjectMasks(self, dataRef):
1032  """Retrieve the bright object masks.
1033 
1034  Returns None on failure.
1035 
1036  Parameters
1037  ----------
1038  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1039  A Butler dataRef.
1040 
1041  Returns
1042  -------
1043  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1044  Bright object mask from the Butler object, or None if it cannot
1045  be retrieved.
1046  """
1047  try:
1048  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1049  except Exception as e:
1050  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1051  return None
1052 
1053  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1054  """Set the bright object masks.
1055 
1056  Parameters
1057  ----------
1058  exposure : `lsst.afw.image.Exposure`
1059  Exposure under consideration.
1060  dataId : `lsst.daf.persistence.dataId`
1061  Data identifier dict for patch.
1062  brightObjectMasks : `lsst.afw.table`
1063  Table of bright objects to mask.
1064  """
1065 
1066  if brightObjectMasks is None:
1067  self.log.warn("Unable to apply bright object mask: none supplied")
1068  return
1069  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1070  mask = exposure.getMaskedImage().getMask()
1071  wcs = exposure.getWcs()
1072  plateScale = wcs.getPixelScale().asArcseconds()
1073 
1074  for rec in brightObjectMasks:
1075  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1076  if rec["type"] == "box":
1077  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1078  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1079  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1080 
1081  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1082  bbox = geom.Box2I(center - halfSize, center + halfSize)
1083 
1084  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1085  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1086  spans = afwGeom.SpanSet(bbox)
1087  elif rec["type"] == "circle":
1088  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1089  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1090  else:
1091  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1092  continue
1093  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1094 
1095  def setInexactPsf(self, mask):
1096  """Set INEXACT_PSF mask plane.
1097 
1098  If any of the input images isn't represented in the coadd (due to
1099  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1100  these pixels.
1101 
1102  Parameters
1103  ----------
1104  mask : `lsst.afw.image.Mask`
1105  Coadded exposure's mask, modified in-place.
1106  """
1107  mask.addMaskPlane("INEXACT_PSF")
1108  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1109  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1110  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1111  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1112  array = mask.getArray()
1113  selected = array & (sensorEdge | clipped | rejected) > 0
1114  array[selected] |= inexactPsf
1115 
1116  @classmethod
1117  def _makeArgumentParser(cls):
1118  """Create an argument parser.
1119  """
1120  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1121  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1122  + cls.ConfigClass().warpType + "Warp",
1123  help="data ID, e.g. --id tract=12345 patch=1,2",
1124  ContainerClass=AssembleCoaddDataIdContainer)
1125  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1126  ContainerClass=SelectDataIdContainer)
1127  return parser
1128 
1129  @staticmethod
1130  def _subBBoxIter(bbox, subregionSize):
1131  """Iterate over subregions of a bbox.
1132 
1133  Parameters
1134  ----------
1135  bbox : `lsst.geom.Box2I`
1136  Bounding box over which to iterate.
1137  subregionSize: `lsst.geom.Extent2I`
1138  Size of sub-bboxes.
1139 
1140  Yields
1141  ------
1142  subBBox : `lsst.geom.Box2I`
1143  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1144  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1145  the edges of ``bbox``, but it will never be empty.
1146  """
1147  if bbox.isEmpty():
1148  raise RuntimeError("bbox %s is empty" % (bbox,))
1149  if subregionSize[0] < 1 or subregionSize[1] < 1:
1150  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1151 
1152  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1153  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1154  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1155  subBBox.clip(bbox)
1156  if subBBox.isEmpty():
1157  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1158  "colShift=%s, rowShift=%s" %
1159  (bbox, subregionSize, colShift, rowShift))
1160  yield subBBox
1161 
1162 
1163 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1164  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1165  """
1166 
1167  def makeDataRefList(self, namespace):
1168  """Make self.refList from self.idList.
1169 
1170  Parameters
1171  ----------
1172  namespace
1173  Results of parsing command-line (with ``butler`` and ``log`` elements).
1174  """
1175  datasetType = namespace.config.coaddName + "Coadd"
1176  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1177 
1178  for dataId in self.idList:
1179  # tract and patch are required
1180  for key in keysCoadd:
1181  if key not in dataId:
1182  raise RuntimeError("--id must include " + key)
1183 
1184  dataRef = namespace.butler.dataRef(
1185  datasetType=datasetType,
1186  dataId=dataId,
1187  )
1188  self.refList.append(dataRef)
1189 
1190 
1191 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1192  """Function to count the number of pixels with a specific mask in a
1193  footprint.
1194 
1195  Find the intersection of mask & footprint. Count all pixels in the mask
1196  that are in the intersection that have bitmask set but do not have
1197  ignoreMask set. Return the count.
1198 
1199  Parameters
1200  ----------
1201  mask : `lsst.afw.image.Mask`
1202  Mask to define intersection region by.
1203  footprint : `lsst.afw.detection.Footprint`
1204  Footprint to define the intersection region by.
1205  bitmask
1206  Specific mask that we wish to count the number of occurances of.
1207  ignoreMask
1208  Pixels to not consider.
1209 
1210  Returns
1211  -------
1212  result : `int`
1213  Count of number of pixels in footprint with specified mask.
1214  """
1215  bbox = footprint.getBBox()
1216  bbox.clip(mask.getBBox(afwImage.PARENT))
1217  fp = afwImage.Mask(bbox)
1218  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1219  footprint.spans.setMask(fp, bitmask)
1220  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1221  (subMask.getArray() & ignoreMask) == 0).sum()
1222 
1223 
1224 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1225  """Configuration parameters for the SafeClipAssembleCoaddTask.
1226  """
1227  assembleMeanCoadd = pexConfig.ConfigurableField(
1228  target=AssembleCoaddTask,
1229  doc="Task to assemble an initial Coadd using the MEAN statistic.",
1230  )
1231  assembleMeanClipCoadd = pexConfig.ConfigurableField(
1232  target=AssembleCoaddTask,
1233  doc="Task to assemble an initial Coadd using the MEANCLIP statistic.",
1234  )
1235  clipDetection = pexConfig.ConfigurableField(
1236  target=SourceDetectionTask,
1237  doc="Detect sources on difference between unclipped and clipped coadd")
1238  minClipFootOverlap = pexConfig.Field(
1239  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1240  dtype=float,
1241  default=0.6
1242  )
1243  minClipFootOverlapSingle = pexConfig.Field(
1244  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1245  "clipped when only one visit overlaps",
1246  dtype=float,
1247  default=0.5
1248  )
1249  minClipFootOverlapDouble = pexConfig.Field(
1250  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1251  "clipped when two visits overlap",
1252  dtype=float,
1253  default=0.45
1254  )
1255  maxClipFootOverlapDouble = pexConfig.Field(
1256  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1257  "considering two visits",
1258  dtype=float,
1259  default=0.15
1260  )
1261  minBigOverlap = pexConfig.Field(
1262  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1263  "when labeling clipped footprints",
1264  dtype=int,
1265  default=100
1266  )
1267 
1268  def setDefaults(self):
1269  """Set default values for clipDetection.
1270 
1271  Notes
1272  -----
1273  The numeric values for these configuration parameters were
1274  empirically determined, future work may further refine them.
1275  """
1276  AssembleCoaddConfig.setDefaults(self)
1277  self.clipDetection.doTempLocalBackground = False
1278  self.clipDetection.reEstimateBackground = False
1279  self.clipDetection.returnOriginalFootprints = False
1280  self.clipDetection.thresholdPolarity = "both"
1281  self.clipDetection.thresholdValue = 2
1282  self.clipDetection.nSigmaToGrow = 2
1283  self.clipDetection.minPixels = 4
1284  self.clipDetection.isotropicGrow = True
1285  self.clipDetection.thresholdType = "pixel_stdev"
1286  self.sigmaClip = 1.5
1287  self.clipIter = 3
1288  self.statistic = "MEAN"
1289  self.assembleMeanCoadd.statistic = 'MEAN'
1290  self.assembleMeanClipCoadd.statistic = 'MEANCLIP'
1291  self.assembleMeanCoadd.doWrite = False
1292  self.assembleMeanClipCoadd.doWrite = False
1293 
1294  def validate(self):
1295  if self.doSigmaClip:
1296  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1297  "Ignoring doSigmaClip.")
1298  self.doSigmaClip = False
1299  if self.statistic != "MEAN":
1300  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1301  "(%s chosen). Please set statistic to MEAN."
1302  % (self.statistic))
1303  AssembleCoaddTask.ConfigClass.validate(self)
1304 
1305 
1306 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1307  """Assemble a coadded image from a set of coadded temporary exposures,
1308  being careful to clip & flag areas with potential artifacts.
1309 
1310  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1311  we clip outliers). The problem with doing this is that when computing the
1312  coadd PSF at a given location, individual visit PSFs from visits with
1313  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1314  In this task, we correct for this behavior by creating a new
1315  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1316  coaddTempExps and the final coadd where
1317 
1318  i. difference imaging suggests that there is an outlier and
1319  ii. this outlier appears on only one or two images.
1320 
1321  Such regions will not contribute to the final coadd. Furthermore, any
1322  routine to determine the coadd PSF can now be cognizant of clipped regions.
1323  Note that the algorithm implemented by this task is preliminary and works
1324  correctly for HSC data. Parameter modifications and or considerable
1325  redesigning of the algorithm is likley required for other surveys.
1326 
1327  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1328  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1329  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1330  if you wish.
1331 
1332  Notes
1333  -----
1334  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1335  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1336  see `baseDebug` for more about ``debug.py`` files.
1337  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1338  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1339  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1340  for further information.
1341 
1342  Examples
1343  --------
1344  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1345  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1346  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1347 
1348  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1349  and filter to be coadded (specified using
1350  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1351  along with a list of coaddTempExps to attempt to coadd (specified using
1352  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1353  Only the coaddTempExps that cover the specified tract and patch will be
1354  coadded. A list of the available optional arguments can be obtained by
1355  calling assembleCoadd.py with the --help command line argument:
1356 
1357  .. code-block:: none
1358 
1359  assembleCoadd.py --help
1360 
1361  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1362  context of multi-band processing, we will generate the HSC-I & -R band
1363  coadds from HSC engineering test data provided in the ci_hsc package.
1364  To begin, assuming that the lsst stack has been already set up, we must
1365  set up the obs_subaru and ci_hsc packages. This defines the environment
1366  variable $CI_HSC_DIR and points at the location of the package. The raw
1367  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1368  the coadds, we must first
1369 
1370  - ``processCcd``
1371  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1372  - ``makeSkyMap``
1373  create a skymap that covers the area of the sky present in the raw exposures
1374  - ``makeCoaddTempExp``
1375  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1376 
1377  We can perform all of these steps by running
1378 
1379  .. code-block:: none
1380 
1381  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1382 
1383  This will produce warped coaddTempExps for each visit. To coadd the
1384  warped data, we call ``assembleCoadd.py`` as follows:
1385 
1386  .. code-block:: none
1387 
1388  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1389  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1390  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1391  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1392  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1393  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1394  --selectId visit=903988 ccd=24
1395 
1396  This will process the HSC-I band data. The results are written in
1397  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1398 
1399  You may also choose to run:
1400 
1401  .. code-block:: none
1402 
1403  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1404  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1405  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1406  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1407  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1408  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1409  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1410  --selectId visit=903346 ccd=12
1411 
1412  to generate the coadd for the HSC-R band if you are interested in following
1413  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1414  """
1415  ConfigClass = SafeClipAssembleCoaddConfig
1416  _DefaultName = "safeClipAssembleCoadd"
1417 
1418  def __init__(self, *args, **kwargs):
1419  AssembleCoaddTask.__init__(self, *args, **kwargs)
1420  schema = afwTable.SourceTable.makeMinimalSchema()
1421  self.makeSubtask("clipDetection", schema=schema)
1422  self.makeSubtask("assembleMeanClipCoadd")
1423  self.makeSubtask("assembleMeanCoadd")
1424 
1425  @utils.inheritDoc(AssembleCoaddTask)
1426  @pipeBase.timeMethod
1427  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1428  """Assemble the coadd for a region.
1429 
1430  Compute the difference of coadds created with and without outlier
1431  rejection to identify coadd pixels that have outlier values in some
1432  individual visits.
1433  Detect clipped regions on the difference image and mark these regions
1434  on the one or two individual coaddTempExps where they occur if there
1435  is significant overlap between the clipped region and a source. This
1436  leaves us with a set of footprints from the difference image that have
1437  been identified as having occured on just one or two individual visits.
1438  However, these footprints were generated from a difference image. It
1439  is conceivable for a large diffuse source to have become broken up
1440  into multiple footprints acrosss the coadd difference in this process.
1441  Determine the clipped region from all overlapping footprints from the
1442  detected sources in each visit - these are big footprints.
1443  Combine the small and big clipped footprints and mark them on a new
1444  bad mask plane.
1445  Generate the coadd using `AssembleCoaddTask.run` without outlier
1446  removal. Clipped footprints will no longer make it into the coadd
1447  because they are marked in the new bad mask plane.
1448 
1449  Notes
1450  -----
1451  args and kwargs are passed but ignored in order to match the call
1452  signature expected by the parent task.
1453  """
1454  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1455  mask = exp.getMaskedImage().getMask()
1456  mask.addMaskPlane("CLIPPED")
1457 
1458  result = self.detectClip(exp, tempExpRefList)
1459 
1460  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1461 
1462  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1463  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1464  # Append big footprints from individual Warps to result.clipSpans
1465  bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1466  result.detectionFootprints, maskClipValue, maskDetValue,
1467  exp.getBBox())
1468  # Create mask of the current clipped footprints
1469  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1470  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1471 
1472  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1473  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1474  maskClip |= maskClipBig
1475 
1476  # Assemble coadd from base class, but ignoring CLIPPED pixels
1477  badMaskPlanes = self.config.badMaskPlanes[:]
1478  badMaskPlanes.append("CLIPPED")
1479  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1480  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1481  result.clipSpans, mask=badPixelMask)
1482 
1483  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1484  """Return an exposure that contains the difference between unclipped
1485  and clipped coadds.
1486 
1487  Generate a difference image between clipped and unclipped coadds.
1488  Compute the difference image by subtracting an outlier-clipped coadd
1489  from an outlier-unclipped coadd. Return the difference image.
1490 
1491  Parameters
1492  ----------
1493  skyInfo : `lsst.pipe.base.Struct`
1494  Patch geometry information, from getSkyInfo
1495  tempExpRefList : `list`
1496  List of data reference to tempExp
1497  imageScalerList : `list`
1498  List of image scalers
1499  weightList : `list`
1500  List of weights
1501 
1502  Returns
1503  -------
1504  exp : `lsst.afw.image.Exposure`
1505  Difference image of unclipped and clipped coadd wrapped in an Exposure
1506  """
1507  coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList,
1508  imageScalerList, weightList).coaddExposure
1509 
1510  coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList,
1511  imageScalerList, weightList).coaddExposure
1512 
1513  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1514  coaddDiff -= coaddClip.getMaskedImage()
1515  exp = afwImage.ExposureF(coaddDiff)
1516  exp.setPsf(coaddMean.getPsf())
1517  return exp
1518 
1519  def detectClip(self, exp, tempExpRefList):
1520  """Detect clipped regions on an exposure and set the mask on the
1521  individual tempExp masks.
1522 
1523  Detect footprints in the difference image after smoothing the
1524  difference image with a Gaussian kernal. Identify footprints that
1525  overlap with one or two input ``coaddTempExps`` by comparing the
1526  computed overlap fraction to thresholds set in the config. A different
1527  threshold is applied depending on the number of overlapping visits
1528  (restricted to one or two). If the overlap exceeds the thresholds,
1529  the footprint is considered "CLIPPED" and is marked as such on the
1530  coaddTempExp. Return a struct with the clipped footprints, the indices
1531  of the ``coaddTempExps`` that end up overlapping with the clipped
1532  footprints, and a list of new masks for the ``coaddTempExps``.
1533 
1534  Parameters
1535  ----------
1536  exp : `lsst.afw.image.Exposure`
1537  Exposure to run detection on.
1538  tempExpRefList : `list`
1539  List of data reference to tempExp.
1540 
1541  Returns
1542  -------
1543  result : `lsst.pipe.base.Struct`
1544  Result struct with components:
1545 
1546  - ``clipFootprints``: list of clipped footprints.
1547  - ``clipIndices``: indices for each ``clippedFootprint`` in
1548  ``tempExpRefList``.
1549  - ``clipSpans``: List of dictionaries containing spanSet lists
1550  to clip. Each element contains the new maskplane name
1551  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1552  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1553  compressed into footprints.
1554  """
1555  mask = exp.getMaskedImage().getMask()
1556  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1557  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1558  # Merge positive and negative together footprints together
1559  fpSet.positive.merge(fpSet.negative)
1560  footprints = fpSet.positive
1561  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1562  ignoreMask = self.getBadPixelMask()
1563 
1564  clipFootprints = []
1565  clipIndices = []
1566  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1567 
1568  # for use by detectClipBig
1569  visitDetectionFootprints = []
1570 
1571  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1572  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1573  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1574 
1575  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1576  for i, warpRef in enumerate(tempExpRefList):
1577  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1578  immediate=True).getMaskedImage().getMask()
1579  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1580  afwImage.PARENT, True)
1581  maskVisitDet &= maskDetValue
1582  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1583  visitDetectionFootprints.append(visitFootprints)
1584 
1585  for j, footprint in enumerate(footprints.getFootprints()):
1586  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1587  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1588 
1589  # build a list of clipped spans for each visit
1590  for j, footprint in enumerate(footprints.getFootprints()):
1591  nPixel = footprint.getArea()
1592  overlap = [] # hold the overlap with each visit
1593  indexList = [] # index of visit in global list
1594  for i in range(len(tempExpRefList)):
1595  ignore = ignoreArr[i, j]
1596  overlapDet = overlapDetArr[i, j]
1597  totPixel = nPixel - ignore
1598 
1599  # If we have more bad pixels than detection skip
1600  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1601  continue
1602  overlap.append(overlapDet/float(totPixel))
1603  indexList.append(i)
1604 
1605  overlap = numpy.array(overlap)
1606  if not len(overlap):
1607  continue
1608 
1609  keep = False # Should this footprint be marked as clipped?
1610  keepIndex = [] # Which tempExps does the clipped footprint belong to
1611 
1612  # If footprint only has one overlap use a lower threshold
1613  if len(overlap) == 1:
1614  if overlap[0] > self.config.minClipFootOverlapSingle:
1615  keep = True
1616  keepIndex = [0]
1617  else:
1618  # This is the general case where only visit should be clipped
1619  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1620  if len(clipIndex) == 1:
1621  keep = True
1622  keepIndex = [clipIndex[0]]
1623 
1624  # Test if there are clipped objects that overlap two different visits
1625  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1626  if len(clipIndex) == 2 and len(overlap) > 3:
1627  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1628  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1629  keep = True
1630  keepIndex = clipIndex
1631 
1632  if not keep:
1633  continue
1634 
1635  for index in keepIndex:
1636  globalIndex = indexList[index]
1637  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1638 
1639  clipIndices.append(numpy.array(indexList)[keepIndex])
1640  clipFootprints.append(footprint)
1641 
1642  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1643  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1644 
1645  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1646  maskClipValue, maskDetValue, coaddBBox):
1647  """Return individual warp footprints for large artifacts and append
1648  them to ``clipList`` in place.
1649 
1650  Identify big footprints composed of many sources in the coadd
1651  difference that may have originated in a large diffuse source in the
1652  coadd. We do this by indentifying all clipped footprints that overlap
1653  significantly with each source in all the coaddTempExps.
1654 
1655  Parameters
1656  ----------
1657  clipList : `list`
1658  List of alt mask SpanSets with clipping information. Modified.
1659  clipFootprints : `list`
1660  List of clipped footprints.
1661  clipIndices : `list`
1662  List of which entries in tempExpClipList each footprint belongs to.
1663  maskClipValue
1664  Mask value of clipped pixels.
1665  maskDetValue
1666  Mask value of detected pixels.
1667  coaddBBox : `lsst.geom.Box`
1668  BBox of the coadd and warps.
1669 
1670  Returns
1671  -------
1672  bigFootprintsCoadd : `list`
1673  List of big footprints
1674  """
1675  bigFootprintsCoadd = []
1676  ignoreMask = self.getBadPixelMask()
1677  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1678  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1679  for footprint in visitFootprints.getFootprints():
1680  footprint.spans.setMask(maskVisitDet, maskDetValue)
1681 
1682  # build a mask of clipped footprints that are in this visit
1683  clippedFootprintsVisit = []
1684  for foot, clipIndex in zip(clipFootprints, clipIndices):
1685  if index not in clipIndex:
1686  continue
1687  clippedFootprintsVisit.append(foot)
1688  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1689  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1690 
1691  bigFootprintsVisit = []
1692  for foot in visitFootprints.getFootprints():
1693  if foot.getArea() < self.config.minBigOverlap:
1694  continue
1695  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1696  if nCount > self.config.minBigOverlap:
1697  bigFootprintsVisit.append(foot)
1698  bigFootprintsCoadd.append(foot)
1699 
1700  for footprint in bigFootprintsVisit:
1701  clippedSpans["CLIPPED"].append(footprint.spans)
1702 
1703  return bigFootprintsCoadd
1704 
1705 
1707  psfMatchedWarps = pipeBase.connectionTypes.Input(
1708  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1709  "Only PSF-Matched Warps make sense for image subtraction. "
1710  "Therefore, they must be an additional declared input."),
1711  name="{inputCoaddName}Coadd_psfMatchedWarp",
1712  storageClass="ExposureF",
1713  dimensions=("tract", "patch", "skymap", "visit"),
1714  deferLoad=True,
1715  multiple=True
1716  )
1717  templateCoadd = pipeBase.connectionTypes.Output(
1718  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1719  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1720  name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1721  storageClass="ExposureF",
1722  dimensions=("tract", "patch", "skymap", "abstract_filter"),
1723  )
1724 
1725  def __init__(self, *, config=None):
1726  super().__init__(config=config)
1727  if not config.assembleStaticSkyModel.doWrite:
1728  self.outputs.remove("templateCoadd")
1729  config.validate()
1730 
1731 
1732 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1733  pipelineConnections=CompareWarpAssembleCoaddConnections):
1734  assembleStaticSkyModel = pexConfig.ConfigurableField(
1735  target=AssembleCoaddTask,
1736  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1737  " naive/first-iteration model of the static sky.",
1738  )
1739  detect = pexConfig.ConfigurableField(
1740  target=SourceDetectionTask,
1741  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1742  )
1743  detectTemplate = pexConfig.ConfigurableField(
1744  target=SourceDetectionTask,
1745  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1746  )
1747  maxNumEpochs = pexConfig.Field(
1748  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1749  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1750  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1751  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1752  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1753  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1754  "than transient and not masked.",
1755  dtype=int,
1756  default=2
1757  )
1758  maxFractionEpochsLow = pexConfig.RangeField(
1759  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1760  "Effective maxNumEpochs = "
1761  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1762  dtype=float,
1763  default=0.4,
1764  min=0., max=1.,
1765  )
1766  maxFractionEpochsHigh = pexConfig.RangeField(
1767  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1768  "Effective maxNumEpochs = "
1769  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1770  dtype=float,
1771  default=0.03,
1772  min=0., max=1.,
1773  )
1774  spatialThreshold = pexConfig.RangeField(
1775  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1776  "temporal criteria. If 0, clip all. If 1, clip none.",
1777  dtype=float,
1778  default=0.5,
1779  min=0., max=1.,
1780  inclusiveMin=True, inclusiveMax=True
1781  )
1782  doScaleWarpVariance = pexConfig.Field(
1783  doc="Rescale Warp variance plane using empirical noise?",
1784  dtype=bool,
1785  default=True,
1786  )
1787  scaleWarpVariance = pexConfig.ConfigurableField(
1788  target=ScaleVarianceTask,
1789  doc="Rescale variance on warps",
1790  )
1791  doPreserveContainedBySource = pexConfig.Field(
1792  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1793  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1794  dtype=bool,
1795  default=True,
1796  )
1797  doPrefilterArtifacts = pexConfig.Field(
1798  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1799  "because they will be excluded anyway. This prevents them from contributing "
1800  "to the outlier epoch count image and potentially being labeled as persistant."
1801  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1802  dtype=bool,
1803  default=True
1804  )
1805  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1806  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1807  dtype=str,
1808  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1809  )
1810  prefilterArtifactsRatio = pexConfig.Field(
1811  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1812  dtype=float,
1813  default=0.05
1814  )
1815 
1816  def setDefaults(self):
1817  AssembleCoaddConfig.setDefaults(self)
1818  self.statistic = 'MEAN'
1820 
1821  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1822  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1823  if "EDGE" in self.badMaskPlanes:
1824  self.badMaskPlanes.remove('EDGE')
1825  self.removeMaskPlanes.append('EDGE')
1826  self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1827  self.assembleStaticSkyModel.warpType = 'psfMatched'
1828  self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1829  self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1830  self.assembleStaticSkyModel.sigmaClip = 2.5
1831  self.assembleStaticSkyModel.clipIter = 3
1832  self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1833  self.assembleStaticSkyModel.doWrite = False
1834  self.detect.doTempLocalBackground = False
1835  self.detect.reEstimateBackground = False
1836  self.detect.returnOriginalFootprints = False
1837  self.detect.thresholdPolarity = "both"
1838  self.detect.thresholdValue = 5
1839  self.detect.minPixels = 4
1840  self.detect.isotropicGrow = True
1841  self.detect.thresholdType = "pixel_stdev"
1842  self.detect.nSigmaToGrow = 0.4
1843  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1844  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1845  self.detectTemplate.nSigmaToGrow = 2.4
1846  self.detectTemplate.doTempLocalBackground = False
1847  self.detectTemplate.reEstimateBackground = False
1848  self.detectTemplate.returnOriginalFootprints = False
1849 
1850  def validate(self):
1851  super().validate()
1852  if self.assembleStaticSkyModel.doNImage:
1853  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1854  "Please set assembleStaticSkyModel.doNImage=False")
1855 
1856  if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1857  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1858  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1859  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1860  "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1861 
1862 
1863 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1864  """Assemble a compareWarp coadded image from a set of warps
1865  by masking artifacts detected by comparing PSF-matched warps.
1866 
1867  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1868  we clip outliers). The problem with doing this is that when computing the
1869  coadd PSF at a given location, individual visit PSFs from visits with
1870  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1871  In this task, we correct for this behavior by creating a new badMaskPlane
1872  'CLIPPED' which marks pixels in the individual warps suspected to contain
1873  an artifact. We populate this plane on the input warps by comparing
1874  PSF-matched warps with a PSF-matched median coadd which serves as a
1875  model of the static sky. Any group of pixels that deviates from the
1876  PSF-matched template coadd by more than config.detect.threshold sigma,
1877  is an artifact candidate. The candidates are then filtered to remove
1878  variable sources and sources that are difficult to subtract such as
1879  bright stars. This filter is configured using the config parameters
1880  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1881  the maximum fraction of epochs that the deviation can appear in and still
1882  be considered an artifact. The spatialThreshold is the maximum fraction of
1883  pixels in the footprint of the deviation that appear in other epochs
1884  (where other epochs is defined by the temporalThreshold). If the deviant
1885  region meets this criteria of having a significant percentage of pixels
1886  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1887  set in the mask. These regions will not contribute to the final coadd.
1888  Furthermore, any routine to determine the coadd PSF can now be cognizant
1889  of clipped regions. Note that the algorithm implemented by this task is
1890  preliminary and works correctly for HSC data. Parameter modifications and
1891  or considerable redesigning of the algorithm is likley required for other
1892  surveys.
1893 
1894  ``CompareWarpAssembleCoaddTask`` sub-classes
1895  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1896  as a subtask to generate the TemplateCoadd (the model of the static sky).
1897 
1898  Notes
1899  -----
1900  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1901  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1902  ``baseDebug`` for more about ``debug.py`` files.
1903 
1904  This task supports the following debug variables:
1905 
1906  - ``saveCountIm``
1907  If True then save the Epoch Count Image as a fits file in the `figPath`
1908  - ``figPath``
1909  Path to save the debug fits images and figures
1910 
1911  For example, put something like:
1912 
1913  .. code-block:: python
1914 
1915  import lsstDebug
1916  def DebugInfo(name):
1917  di = lsstDebug.getInfo(name)
1918  if name == "lsst.pipe.tasks.assembleCoadd":
1919  di.saveCountIm = True
1920  di.figPath = "/desired/path/to/debugging/output/images"
1921  return di
1922  lsstDebug.Info = DebugInfo
1923 
1924  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1925  ``--debug`` flag. Some subtasks may have their own debug variables;
1926  see individual Task documentation.
1927 
1928  Examples
1929  --------
1930  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1931  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1932  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1933  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1934  and filter to be coadded (specified using
1935  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1936  along with a list of coaddTempExps to attempt to coadd (specified using
1937  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1938  Only the warps that cover the specified tract and patch will be coadded.
1939  A list of the available optional arguments can be obtained by calling
1940  ``assembleCoadd.py`` with the ``--help`` command line argument:
1941 
1942  .. code-block:: none
1943 
1944  assembleCoadd.py --help
1945 
1946  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1947  context of multi-band processing, we will generate the HSC-I & -R band
1948  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1949  To begin, assuming that the lsst stack has been already set up, we must
1950  set up the ``obs_subaru`` and ``ci_hsc`` packages.
1951  This defines the environment variable ``$CI_HSC_DIR`` and points at the
1952  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1953  directory. To begin assembling the coadds, we must first
1954 
1955  - processCcd
1956  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1957  - makeSkyMap
1958  create a skymap that covers the area of the sky present in the raw exposures
1959  - makeCoaddTempExp
1960  warp the individual calibrated exposures to the tangent plane of the coadd
1961 
1962  We can perform all of these steps by running
1963 
1964  .. code-block:: none
1965 
1966  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1967 
1968  This will produce warped ``coaddTempExps`` for each visit. To coadd the
1969  warped data, we call ``assembleCoadd.py`` as follows:
1970 
1971  .. code-block:: none
1972 
1973  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1974  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1975  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1976  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1977  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1978  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1979  --selectId visit=903988 ccd=24
1980 
1981  This will process the HSC-I band data. The results are written in
1982  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1983  """
1984  ConfigClass = CompareWarpAssembleCoaddConfig
1985  _DefaultName = "compareWarpAssembleCoadd"
1986 
1987  def __init__(self, *args, **kwargs):
1988  AssembleCoaddTask.__init__(self, *args, **kwargs)
1989  self.makeSubtask("assembleStaticSkyModel")
1990  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1991  self.makeSubtask("detect", schema=detectionSchema)
1992  if self.config.doPreserveContainedBySource:
1993  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1994  if self.config.doScaleWarpVariance:
1995  self.makeSubtask("scaleWarpVariance")
1996 
1997  @utils.inheritDoc(AssembleCoaddTask)
1998  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
1999  """
2000  Generate a templateCoadd to use as a naive model of static sky to
2001  subtract from PSF-Matched warps.
2002 
2003  Returns
2004  -------
2005  result : `lsst.pipe.base.Struct`
2006  Result struct with components:
2007 
2008  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2009  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2010  """
2011  # Ensure that psfMatchedWarps are used as input warps for template generation
2012  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2013  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2014 
2015  # Because subtasks don't have connections we have to make one.
2016  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2017  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2018  if self.config.assembleStaticSkyModel.doWrite:
2019  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2020  # Remove template coadd from both subtask's and main tasks outputs,
2021  # because it is handled by the subtask as `coaddExposure`
2022  del outputRefs.templateCoadd
2023  del staticSkyModelOutputRefs.templateCoadd
2024 
2025  # A PSF-Matched nImage does not exist as a dataset type
2026  if 'nImage' in staticSkyModelOutputRefs.keys():
2027  del staticSkyModelOutputRefs.nImage
2028 
2029  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2030  staticSkyModelOutputRefs)
2031  if templateCoadd is None:
2032  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2033 
2034  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2035  nImage=templateCoadd.nImage,
2036  warpRefList=templateCoadd.warpRefList,
2037  imageScalerList=templateCoadd.imageScalerList,
2038  weightList=templateCoadd.weightList)
2039 
2040  @utils.inheritDoc(AssembleCoaddTask)
2041  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2042  """
2043  Generate a templateCoadd to use as a naive model of static sky to
2044  subtract from PSF-Matched warps.
2045 
2046  Returns
2047  -------
2048  result : `lsst.pipe.base.Struct`
2049  Result struct with components:
2050 
2051  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2052  - ``nImage``: N Image (``lsst.afw.image.Image``)
2053  """
2054  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2055  if templateCoadd is None:
2056  raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2057 
2058  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2059  nImage=templateCoadd.nImage,
2060  warpRefList=templateCoadd.warpRefList,
2061  imageScalerList=templateCoadd.imageScalerList,
2062  weightList=templateCoadd.weightList)
2063 
2064  def _noTemplateMessage(self, warpType):
2065  warpName = (warpType[0].upper() + warpType[1:])
2066  message = """No %(warpName)s warps were found to build the template coadd which is
2067  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2068  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2069  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2070 
2071  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2072  another algorithm like:
2073 
2074  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2075  config.assemble.retarget(SafeClipAssembleCoaddTask)
2076  """ % {"warpName": warpName}
2077  return message
2078 
2079  @utils.inheritDoc(AssembleCoaddTask)
2080  @pipeBase.timeMethod
2081  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2082  supplementaryData, *args, **kwargs):
2083  """Assemble the coadd.
2084 
2085  Find artifacts and apply them to the warps' masks creating a list of
2086  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2087  plane. Then pass these alternative masks to the base class's `run`
2088  method.
2089 
2090  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2091  that must contain a ``templateCoadd`` that serves as the
2092  model of the static sky.
2093  """
2094 
2095  # Check and match the order of the supplementaryData
2096  # (PSF-matched) inputs to the order of the direct inputs,
2097  # so that the artifact mask is applied to the right warp
2098  dataIds = [ref.dataId for ref in tempExpRefList]
2099  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2100 
2101  if dataIds != psfMatchedDataIds:
2102  self.log.info("Reordering and or/padding PSF-matched visit input list")
2103  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2104  psfMatchedDataIds, dataIds)
2105  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2106  psfMatchedDataIds, dataIds)
2107 
2108  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2109  spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2110  supplementaryData.warpRefList,
2111  supplementaryData.imageScalerList)
2112 
2113  badMaskPlanes = self.config.badMaskPlanes[:]
2114  badMaskPlanes.append("CLIPPED")
2115  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2116 
2117  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2118  spanSetMaskList, mask=badPixelMask)
2119 
2120  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2121  # Psf-Matching moves the real edge inwards
2122  self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2123  return result
2124 
2125  def applyAltEdgeMask(self, mask, altMaskList):
2126  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2127 
2128  Parameters
2129  ----------
2130  mask : `lsst.afw.image.Mask`
2131  Original mask.
2132  altMaskList : `list`
2133  List of Dicts containing ``spanSet`` lists.
2134  Each element contains the new mask plane name (e.g. "CLIPPED
2135  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2136  the mask.
2137  """
2138  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2139  for visitMask in altMaskList:
2140  if "EDGE" in visitMask:
2141  for spanSet in visitMask['EDGE']:
2142  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2143 
2144  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2145  """Find artifacts.
2146 
2147  Loop through warps twice. The first loop builds a map with the count
2148  of how many epochs each pixel deviates from the templateCoadd by more
2149  than ``config.chiThreshold`` sigma. The second loop takes each
2150  difference image and filters the artifacts detected in each using
2151  count map to filter out variable sources and sources that are
2152  difficult to subtract cleanly.
2153 
2154  Parameters
2155  ----------
2156  templateCoadd : `lsst.afw.image.Exposure`
2157  Exposure to serve as model of static sky.
2158  tempExpRefList : `list`
2159  List of data references to warps.
2160  imageScalerList : `list`
2161  List of image scalers.
2162 
2163  Returns
2164  -------
2165  altMasks : `list`
2166  List of dicts containing information about CLIPPED
2167  (i.e., artifacts), NO_DATA, and EDGE pixels.
2168  """
2169 
2170  self.log.debug("Generating Count Image, and mask lists.")
2171  coaddBBox = templateCoadd.getBBox()
2172  slateIm = afwImage.ImageU(coaddBBox)
2173  epochCountImage = afwImage.ImageU(coaddBBox)
2174  nImage = afwImage.ImageU(coaddBBox)
2175  spanSetArtifactList = []
2176  spanSetNoDataMaskList = []
2177  spanSetEdgeList = []
2178  badPixelMask = self.getBadPixelMask()
2179 
2180  # mask of the warp diffs should = that of only the warp
2181  templateCoadd.mask.clearAllMaskPlanes()
2182 
2183  if self.config.doPreserveContainedBySource:
2184  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2185  else:
2186  templateFootprints = None
2187 
2188  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2189  warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2190  if warpDiffExp is not None:
2191  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2192  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2193  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2194  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2195  fpSet.positive.merge(fpSet.negative)
2196  footprints = fpSet.positive
2197  slateIm.set(0)
2198  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2199 
2200  # Remove artifacts due to defects before they contribute to the epochCountImage
2201  if self.config.doPrefilterArtifacts:
2202  spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2203  for spans in spanSetList:
2204  spans.setImage(slateIm, 1, doClip=True)
2205  epochCountImage += slateIm
2206 
2207  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2208  # undergo a second convolution. Pixels with data in the direct warp
2209  # but not in the PSF-matched warp will not have their artifacts detected.
2210  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2211  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2212  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2213  nansMask.setXY0(warpDiffExp.getXY0())
2214  edgeMask = warpDiffExp.mask
2215  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2216  edgeMask.getPlaneBitMask("EDGE")).split()
2217  else:
2218  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2219  # In this case, mask the whole epoch
2220  nansMask = afwImage.MaskX(coaddBBox, 1)
2221  spanSetList = []
2222  spanSetEdgeMask = []
2223 
2224  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2225 
2226  spanSetNoDataMaskList.append(spanSetNoDataMask)
2227  spanSetArtifactList.append(spanSetList)
2228  spanSetEdgeList.append(spanSetEdgeMask)
2229 
2230  if lsstDebug.Info(__name__).saveCountIm:
2231  path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2232  epochCountImage.writeFits(path)
2233 
2234  for i, spanSetList in enumerate(spanSetArtifactList):
2235  if spanSetList:
2236  filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2237  templateFootprints)
2238  spanSetArtifactList[i] = filteredSpanSetList
2239 
2240  altMasks = []
2241  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2242  altMasks.append({'CLIPPED': artifacts,
2243  'NO_DATA': noData,
2244  'EDGE': edge})
2245  return altMasks
2246 
2247  def prefilterArtifacts(self, spanSetList, exp):
2248  """Remove artifact candidates covered by bad mask plane.
2249 
2250  Any future editing of the candidate list that does not depend on
2251  temporal information should go in this method.
2252 
2253  Parameters
2254  ----------
2255  spanSetList : `list`
2256  List of SpanSets representing artifact candidates.
2257  exp : `lsst.afw.image.Exposure`
2258  Exposure containing mask planes used to prefilter.
2259 
2260  Returns
2261  -------
2262  returnSpanSetList : `list`
2263  List of SpanSets with artifacts.
2264  """
2265  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2266  goodArr = (exp.mask.array & badPixelMask) == 0
2267  returnSpanSetList = []
2268  bbox = exp.getBBox()
2269  x0, y0 = exp.getXY0()
2270  for i, span in enumerate(spanSetList):
2271  y, x = span.clippedTo(bbox).indices()
2272  yIndexLocal = numpy.array(y) - y0
2273  xIndexLocal = numpy.array(x) - x0
2274  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2275  if goodRatio > self.config.prefilterArtifactsRatio:
2276  returnSpanSetList.append(span)
2277  return returnSpanSetList
2278 
2279  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2280  """Filter artifact candidates.
2281 
2282  Parameters
2283  ----------
2284  spanSetList : `list`
2285  List of SpanSets representing artifact candidates.
2286  epochCountImage : `lsst.afw.image.Image`
2287  Image of accumulated number of warpDiff detections.
2288  nImage : `lsst.afw.image.Image`
2289  Image of the accumulated number of total epochs contributing.
2290 
2291  Returns
2292  -------
2293  maskSpanSetList : `list`
2294  List of SpanSets with artifacts.
2295  """
2296 
2297  maskSpanSetList = []
2298  x0, y0 = epochCountImage.getXY0()
2299  for i, span in enumerate(spanSetList):
2300  y, x = span.indices()
2301  yIdxLocal = [y1 - y0 for y1 in y]
2302  xIdxLocal = [x1 - x0 for x1 in x]
2303  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2304  totalN = nImage.array[yIdxLocal, xIdxLocal]
2305 
2306  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2307  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2308  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2309  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2310  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2311  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2312  & (outlierN <= effectiveMaxNumEpochs))
2313  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2314  if percentBelowThreshold > self.config.spatialThreshold:
2315  maskSpanSetList.append(span)
2316 
2317  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2318  # If a candidate is contained by a footprint on the template coadd, do not clip
2319  filteredMaskSpanSetList = []
2320  for span in maskSpanSetList:
2321  doKeep = True
2322  for footprint in footprintsToExclude.positive.getFootprints():
2323  if footprint.spans.contains(span):
2324  doKeep = False
2325  break
2326  if doKeep:
2327  filteredMaskSpanSetList.append(span)
2328  maskSpanSetList = filteredMaskSpanSetList
2329 
2330  return maskSpanSetList
2331 
2332  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2333  """Fetch a warp from the butler and return a warpDiff.
2334 
2335  Parameters
2336  ----------
2337  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2338  Butler dataRef for the warp.
2339  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2340  An image scaler object.
2341  templateCoadd : `lsst.afw.image.Exposure`
2342  Exposure to be substracted from the scaled warp.
2343 
2344  Returns
2345  -------
2346  warp : `lsst.afw.image.Exposure`
2347  Exposure of the image difference between the warp and template.
2348  """
2349 
2350  # If the PSF-Matched warp did not exist for this direct warp
2351  # None is holding its place to maintain order in Gen 3
2352  if warpRef is None:
2353  return None
2354  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2355  warpName = self.getTempExpDatasetName('psfMatched')
2356  if not isinstance(warpRef, DeferredDatasetHandle):
2357  if not warpRef.datasetExists(warpName):
2358  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2359  return None
2360  warp = warpRef.get(datasetType=warpName, immediate=True)
2361  # direct image scaler OK for PSF-matched Warp
2362  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2363  mi = warp.getMaskedImage()
2364  if self.config.doScaleWarpVariance:
2365  try:
2366  self.scaleWarpVariance.run(mi)
2367  except Exception as exc:
2368  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2369  mi -= templateCoadd.getMaskedImage()
2370  return warp
2371 
2372  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2373  """Return a path to which to write debugging output.
2374 
2375  Creates a hyphen-delimited string of dataId values for simple filenames.
2376 
2377  Parameters
2378  ----------
2379  prefix : `str`
2380  Prefix for filename.
2381  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2382  Butler dataRef to make the path from.
2383  coaddLevel : `bool`, optional.
2384  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2385  'filter', but no 'visit').
2386 
2387  Returns
2388  -------
2389  result : `str`
2390  Path for debugging output.
2391  """
2392  if coaddLevel:
2393  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2394  else:
2395  keys = warpRef.dataId.keys()
2396  keyList = sorted(keys, reverse=True)
2397  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2398  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2399  return os.path.join(directory, filename)
2400 
2401 
2402 def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2403  """Match the order of one list to another, padding if necessary
2404 
2405  Parameters
2406  ----------
2407  inputList : list
2408  List to be reordered and padded. Elements can be any type.
2409  inputKeys : iterable
2410  Iterable of values to be compared with outputKeys.
2411  Length must match `inputList`
2412  outputKeys : iterable
2413  Iterable of values to be compared with inputKeys.
2414  padWith :
2415  Any value to be inserted where inputKey not in outputKeys
2416 
2417  Returns
2418  -------
2419  list
2420  Copy of inputList reordered per outputKeys and padded with `padWith`
2421  so that the length matches length of outputKeys.
2422  """
2423  outputList = []
2424  for d in outputKeys:
2425  if d in inputKeys:
2426  outputList.append(inputList[inputKeys.index(d)])
2427  else:
2428  outputList.append(padWith)
2429  return outputList
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.filterArtifacts
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
Definition: assembleCoadd.py:2279
lsst::afw::image
lsst.pipe.tasks.assembleCoadd.getTempExpRefList
def getTempExpRefList(self, patchRef, calExpRefList)
Definition: assembleCoadd.py:581
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig
Definition: assembleCoadd.py:1733
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig
Definition: assembleCoadd.py:1224
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.makeSupplementaryData
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
Definition: assembleCoadd.py:2041
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Definition: assembleCoadd.py:1427
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
Definition: assembleCoadd.py:2081
lsst.pipe.tasks.assembleCoadd.assembleSubregion
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Definition: assembleCoadd.py:855
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.validate
def validate(self)
Definition: assembleCoadd.py:1850
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.__init__
def __init__(self, *args, **kwargs)
Definition: assembleCoadd.py:1418
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask
Definition: assembleCoadd.py:1306
lsst.pipe.tasks.assembleCoadd.processResults
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
Definition: assembleCoadd.py:518
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.clipDetection
clipDetection
Definition: assembleCoadd.py:1235
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.findArtifacts
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Definition: assembleCoadd.py:2144
lsst.pipe.tasks.coaddBase.makeSkyInfo
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.statistic
statistic
Definition: assembleCoadd.py:1818
lsst.pipe.tasks.assembleCoadd.AssembleCoaddConnections
Definition: assembleCoadd.py:55
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._noTemplateMessage
def _noTemplateMessage(self, warpType)
Definition: assembleCoadd.py:2064
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.detectTemplate
detectTemplate
Definition: assembleCoadd.py:1743
lsst.pipe.tasks.assembleCoadd.reorderAndPadList
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: assembleCoadd.py:2402
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.warpType
warpType
Definition: assembleCoadd.py:1856
lsst.pipe.tasks.assembleCoadd.removeMaskPlanes
def removeMaskPlanes(self, maskedImage)
Definition: assembleCoadd.py:932
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.detectClip
def detectClip(self, exp, tempExpRefList)
Definition: assembleCoadd.py:1519
lsst.pipe.tasks.assembleCoadd.assembleMetadata
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Definition: assembleCoadd.py:794
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.sigmaClip
sigmaClip
Definition: assembleCoadd.py:1286
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.assembleMeanClipCoadd
assembleMeanClipCoadd
Definition: assembleCoadd.py:1231
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask
Definition: assembleCoadd.py:1863
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConnections.__init__
def __init__(self, *config=None)
Definition: assembleCoadd.py:1725
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.prefilterArtifacts
def prefilterArtifacts(self, spanSetList, exp)
Definition: assembleCoadd.py:2247
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._readAndComputeWarpDiff
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Definition: assembleCoadd.py:2332
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:713
lsst.pipe.tasks.assembleCoadd.prepareStats
def prepareStats(self, mask=None)
Definition: assembleCoadd.py:680
lsstDebug::Info
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.detect
detect
Definition: assembleCoadd.py:1739
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.doUsePsfMatchedPolygons
doUsePsfMatchedPolygons
Definition: assembleCoadd.py:1819
lsst.pipe.tasks.assembleCoadd.makeSupplementaryData
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
Definition: assembleCoadd.py:538
lsst::afw::table
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConnections
Definition: assembleCoadd.py:1706
lsst::utils
lsst.pipe.tasks.assembleCoadd.setInexactPsf
def setInexactPsf(self, mask)
Definition: assembleCoadd.py:1095
lsst::pex::config
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.assembleMeanCoadd
assembleMeanCoadd
Definition: assembleCoadd.py:1227
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.buildDifferenceImage
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Definition: assembleCoadd.py:1483
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.doSigmaClip
doSigmaClip
Definition: assembleCoadd.py:1298
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddTask.detectClipBig
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Definition: assembleCoadd.py:1645
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.applyAltEdgeMask
def applyAltEdgeMask(self, mask, altMaskList)
Definition: assembleCoadd.py:2125
lsst::afw::detection
lsst.pipe.tasks.coaddHelpers.getGroupDataRef
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
lsst::geom
lsst.pipe.tasks.assembleCoadd.countMaskFromFootprint
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Definition: assembleCoadd.py:1191
lsst.pipe.tasks.assembleCoadd.readBrightObjectMasks
def readBrightObjectMasks(self, dataRef)
Definition: assembleCoadd.py:1031
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.setDefaults
def setDefaults(self)
Definition: assembleCoadd.py:1268
lsst.pipe.tasks.assembleCoadd.AssembleCoaddDataIdContainer
Definition: assembleCoadd.py:1163
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.validate
def validate(self)
Definition: assembleCoadd.py:1294
lsst.pipe.tasks.assembleCoadd.prepareInputs
def prepareInputs(self, refList)
Definition: assembleCoadd.py:605
lsst::afw::math
lsst.pipe.tasks.coaddBase.makeCoaddSuffix
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:336
lsst.pipe.tasks.assembleCoadd.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:557
Point< int, 2 >
lsst.pipe.tasks.coaddBase.CoaddBaseTask
Base class for coaddition.
Definition: coaddBase.py:131
lsst.pipe.tasks.assembleCoadd.setBrightObjectMasks
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
Definition: assembleCoadd.py:1053
lsst::geom::Box2I
lsst::pex::exceptions
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask._dataRef2DebugPath
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Definition: assembleCoadd.py:2372
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.clipIter
clipIter
Definition: assembleCoadd.py:1287
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.__init__
def __init__(self, *args, **kwargs)
Definition: assembleCoadd.py:1987
lsst.pipe.tasks.assembleCoadd.SafeClipAssembleCoaddConfig.statistic
statistic
Definition: assembleCoadd.py:1288
lsst::geom::Box2D
lsst.pipe::base
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.assembleStaticSkyModel
assembleStaticSkyModel
Definition: assembleCoadd.py:1734
lsst::log
lsst.pipe.tasks.assembleCoadd.applyAltMaskPlanes
def applyAltMaskPlanes(self, mask, altMaskSpans)
Definition: assembleCoadd.py:977
lsst::meas::algorithms
lsst.pipe.tasks.coaddHelpers.groupPatchExposures
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:59
Extent< int, 2 >
lsst::coadd::utils
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig.setDefaults
def setDefaults(self)
Definition: assembleCoadd.py:1816
lsst.pipe.tasks.assembleCoadd.AssembleCoaddDataIdContainer.makeDataRefList
def makeDataRefList(self, namespace)
Definition: assembleCoadd.py:1167
lsst.pipe.tasks.assembleCoadd.shrinkValidPolygons
def shrinkValidPolygons(self, coaddInputs)
Definition: assembleCoadd.py:1009
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:1998
lsst::afw::geom