lsst.pipe.tasks  13.0-29-g7046ce1+1
 All Classes Namespaces Files Functions Variables Groups Pages
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import numpy
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.afw.detection as afwDetect
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage import InterpImageTask
39 from .matchBackgrounds import MatchBackgroundsTask
40 from .scaleZeroPoint import ScaleZeroPointTask
41 from .coaddHelpers import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms import SourceDetectionTask
43 
44 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask"]
45 
46 
47 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
48  """!
49 \anchor AssembleCoaddConfig_
50 
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
52  """
53  subregionSize = pexConfig.ListField(
54  dtype=int,
55  doc="Width, height of stack subregion size; "
56  "make small enough that a full stack of images will fit into memory at once.",
57  length=2,
58  default=(2000, 2000),
59  )
60  doSigmaClip = pexConfig.Field(
61  dtype=bool,
62  doc="Perform sigma clipped outlier rejection? If False then compute a simple mean.",
63  default=True,
64  )
65  sigmaClip = pexConfig.Field(
66  dtype=float,
67  doc="Sigma for outlier rejection; ignored if doSigmaClip false.",
68  default=3.0,
69  )
70  clipIter = pexConfig.Field(
71  dtype=int,
72  doc="Number of iterations of outlier rejection; ignored if doSigmaClip false.",
73  default=2,
74  )
75  scaleZeroPoint = pexConfig.ConfigurableField(
76  target=ScaleZeroPointTask,
77  doc="Task to adjust the photometric zero point of the coadd temp exposures",
78  )
79  doInterp = pexConfig.Field(
80  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
81  dtype=bool,
82  default=True,
83  )
84  interpImage = pexConfig.ConfigurableField(
85  target=InterpImageTask,
86  doc="Task to interpolate (and extrapolate) over NaN pixels",
87  )
88  matchBackgrounds = pexConfig.ConfigurableField(
89  target=MatchBackgroundsTask,
90  doc="Task to match backgrounds",
91  )
92  maxMatchResidualRatio = pexConfig.Field(
93  doc="Maximum ratio of the mean squared error of the background matching model to the variance "
94  "of the difference in backgrounds",
95  dtype=float,
96  default=1.1
97  )
98  maxMatchResidualRMS = pexConfig.Field(
99  doc="Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
100  dtype=float,
101  default=1.0
102  )
103  doWrite = pexConfig.Field(
104  doc="Persist coadd?",
105  dtype=bool,
106  default=True,
107  )
108  doMatchBackgrounds = pexConfig.Field(
109  doc="Match backgrounds of coadd temp exposures before coadding them? "
110  "If False, the coadd temp expsosures must already have been background subtracted or matched",
111  dtype=bool,
112  default=True,
113  )
114  autoReference = pexConfig.Field(
115  doc="Automatically select the coadd temp exposure to use as a reference for background matching? "
116  "Ignored if doMatchBackgrounds false. "
117  "If False you must specify the reference temp exposure as the data Id",
118  dtype=bool,
119  default=True,
120  )
121  maskPropagationThresholds = pexConfig.DictField(
122  keytype=str,
123  itemtype=float,
124  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
125  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
126  "would have contributed exceeds this value."),
127  default={"SAT": 0.1},
128  )
129  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["CROSSTALK", "NOT_DEBLENDED"],
130  doc="Mask planes to remove before coadding")
131  #
132  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
133  # To make this useful you *must* also configure the flags.pixel algorithm, for example
134  # by adding
135  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
136  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
137  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
138  #
139  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
140  doc="Set mask and flag bits for bright objects?")
141  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
142  doc="Name of mask bit used for bright objects")
143 
144  def setDefaults(self):
145  CoaddBaseTask.ConfigClass.setDefaults(self)
146  self.badMaskPlanes = ["NO_DATA", "BAD", "CR", ]
147 
148 
149 # \addtogroup LSST_task_documentation
150 # \{
151 # \page AssembleCoaddTask
152 # \ref AssembleCoaddTask_ "AssembleCoaddTask"
153 # \copybrief AssembleCoaddTask
154 # \}
155 
156 class AssembleCoaddTask(CoaddBaseTask):
157  """!
158 \anchor AssembleCoaddTask_
159 
160 \brief Assemble a coadded image from a set of coadded temporary exposures.
161 
162 \section pipe_tasks_assembleCoadd_Contents Contents
163  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
164  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
165  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
166  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
167  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
168  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
169 
170 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
171 
172 \copybrief AssembleCoaddTask_
173 
174 We want to assemble a coadded image from a set of coadded temporary exposures (coaddTempExps).
175 Each input coaddTempExp covers a patch on the sky and corresponds to a single run/visit/exposure of the
176 covered patch. We provide the task with a list of coaddTempExps (selectDataList) from which it selects
177 coaddTempExps that cover the specified patch (pointed at by dataRef).
178 Each coaddTempExp that goes into a coadd will typically have an independent photometric zero-point.
179 Therefore, we must scale each coaddTempExp to set it to a common photometric zeropoint. By default, each
180 coaddTempExp has backgrounds and hence will require config.doMatchBackgrounds=True.
181 When background matching is enabled, the task may be configured to automatically select a reference exposure
182 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
183 coaddTempExp (dataset type coaddName + 'Coadd_tempExp') which is used as the reference exposure.
184 The coadd is computed as a mean with optional outlier rejection.
185 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, coaddTempExps can have bad 'NaN'
186 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
187 
188 AssembleCoaddTask uses several sub-tasks. These are
189 <DL>
190  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
191  <DD> create and use an imageScaler object to scale the photometric zeropoint for each coaddTempExp</DD>
192  <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
193  <DD> match background in a coaddTempExp to a reference exposure (and select the reference exposure if one is
194  not provided).</DD>
195  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
196  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
197 </DL>
198 You can retarget these subtasks if you wish.
199 
200 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
201 \copydoc \_\_init\_\_
202 
203 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
204 \copydoc run
205 
206 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
207 See \ref AssembleCoaddConfig_
208 
209 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
210 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
211 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
212 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
213 the documetation for the subtasks for further information.
214 
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
216 
217 AssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image. The AssembleCoaddTask
218 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
219 a data reference to the tract patch and filter to be coadded (specified using
220 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
221 coaddTempExps to attempt to coadd (specified using
222 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the coaddTempExps
223 that cover the specified tract and patch will be coadded. A list of the available optional
224 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
225 \code
226 assembleCoadd.py --help
227 \endcode
228 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
229 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
230 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
231 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
232 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
233 <DL>
234  <DT>processCcd</DT>
235  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
236  <DT>makeSkyMap</DT>
237  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
238  <DT>makeCoaddTempExp</DT>
239  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
240 </DL>
241 We can perform all of these steps by running
242 \code
243 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
244 \endcode
245 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py as
246 follows:
247 \code
248 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
249 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
250 You may also choose to run:
251 \code
252 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
253 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
254 \endcode
255 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
256 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
257 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
258  """
259  ConfigClass = AssembleCoaddConfig
260  _DefaultName = "assembleCoadd"
261 
262  def __init__(self, *args, **kwargs):
263  """!
264  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
265  \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
266  """
267  CoaddBaseTask.__init__(self, *args, **kwargs)
268  self.makeSubtask("interpImage")
269  self.makeSubtask("matchBackgrounds")
270  self.makeSubtask("scaleZeroPoint")
271 
272  if self.config.doMaskBrightObjects:
273  mask = afwImage.MaskU()
274  try:
275  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
276  except pexExceptions.LsstCppException:
277  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
278  mask.getMaskPlaneDict().keys())
279  del mask
280 
281  @pipeBase.timeMethod
282  def run(self, dataRef, selectDataList=[]):
283  """!
284  \brief Assemble a coadd from a set of coaddTempExp
285 
286  Coadd a set of coaddTempExps. Compute weights to be applied to each coaddTempExp and find scalings to
287  match the photometric zeropoint to a reference coaddTempExp. Optionally, match backgrounds across
288  coaddTempExps if the background has not already been removed. Assemble the coaddTempExps using
289  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
290  exposure.
291 
292  \anchor runParams
293  \param[in] dataRef: Data reference defining the patch for coaddition and the reference coaddTempExp
294  (if config.autoReference=False). Used to access the following data products:
295  - [in] self.config.coaddName + "Coadd_skyMap"
296  - [in] self.config.coaddName + "Coadd_tempExp" (optionally)
297  - [out] self.config.coaddName + "Coadd"
298  \param[in] selectDataList[in]: List of data references to coaddTempExps. Data to be coadded will be
299  selected from this list based on overlap with the patch defined by dataRef.
300 
301  \return a pipeBase.Struct with fields:
302  - coaddExposure: coadded exposure
303  """
304  skyInfo = self.getSkyInfo(dataRef)
305  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
306  if len(calExpRefList) == 0:
307  self.log.warn("No exposures to coadd")
308  return
309  self.log.info("Coadding %d exposures", len(calExpRefList))
310 
311  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
312  inputData = self.prepareInputs(tempExpRefList)
313  self.log.info("Found %d %s", len(inputData.tempExpRefList), self.getTempExpDatasetName())
314  if len(inputData.tempExpRefList) == 0:
315  self.log.warn("No coadd temporary exposures found")
316  return
317  if self.config.doMatchBackgrounds:
318  refImageScaler = self.getBackgroundReferenceScaler(dataRef)
319  inputData = self.backgroundMatching(inputData, dataRef, refImageScaler)
320  if len(inputData.tempExpRefList) == 0:
321  self.log.warn("No valid background models")
322  return
323 
324  coaddExp = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
325  inputData.weightList,
326  inputData.backgroundInfoList if self.config.doMatchBackgrounds else None,
327  doClip=self.config.doSigmaClip)
328  if self.config.doMatchBackgrounds:
329  self.addBackgroundMatchingMetadata(coaddExp, inputData.tempExpRefList,
330  inputData.backgroundInfoList)
331 
332  if self.config.doInterp:
333  self.interpImage.run(coaddExp.getMaskedImage(), planeName="NO_DATA")
334  # The variance must be positive; work around for DM-3201.
335  varArray = coaddExp.getMaskedImage().getVariance().getArray()
336  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
337 
338  if self.config.doMaskBrightObjects:
339  brightObjectMasks = self.readBrightObjectMasks(dataRef)
340  self.setBrightObjectMasks(coaddExp, dataRef.dataId, brightObjectMasks)
341 
342  if self.config.doWrite:
343  self.writeCoaddOutput(dataRef, coaddExp)
344 
345  return pipeBase.Struct(coaddExposure=coaddExp)
346 
347  def getTempExpRefList(self, patchRef, calExpRefList):
348  """!
349  \brief Generate list of coaddTempExp data references corresponding to exposures that lie within the
350  patch to be coadded.
351 
352  \param[in] patchRef: Data reference for patch
353  \param[in] calExpRefList: List of data references for input calexps
354  \return List of coaddTempExp data references
355  """
356  butler = patchRef.getButler()
357  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
358  self.getTempExpDatasetName())
359  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(), g, groupData.keys) for
360  g in groupData.groups.keys()]
361  return tempExpRefList
362 
363  def getBackgroundReferenceScaler(self, dataRef):
364  """!
365  \brief Construct an image scaler for the background reference frame
366 
367  Each coaddTempExp has a different background level. A reference background level must be chosen before
368  coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
369  this routine is a no-op and None is returned. Otherwise, use the
370  \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
371  reference image and return it.
372 
373  \param[in] dataRef: Data reference for the background reference frame, or None
374  \return image scaler, or None
375  """
376  if self.config.autoReference:
377  return None
378 
379  # We've been given the data reference
380  dataset = self.getTempExpDatasetName()
381  if not dataRef.datasetExists(dataset):
382  raise RuntimeError("Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
383 
384  refExposure = dataRef.get(self.getTempExpDatasetName(), immediate=True)
385  refImageScaler = self.scaleZeroPoint.computeImageScaler(
386  exposure=refExposure,
387  dataRef=dataRef,
388  )
389  return refImageScaler
390 
391  def prepareInputs(self, refList):
392  """!
393  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
394  for the photometric zero point.
395 
396  Each coaddTempExp has its own photometric zeropoint and background variance. Before coadding these
397  coaddTempExps together, compute a scale factor to normalize the photometric zeropoint and compute the
398  weight for each coaddTempExp.
399 
400  \param[in] refList: List of data references to tempExp
401  \return Struct:
402  - tempExprefList: List of data references to tempExp
403  - weightList: List of weightings
404  - imageScalerList: List of image scalers
405  """
406  statsCtrl = afwMath.StatisticsControl()
407  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
408  statsCtrl.setNumIter(self.config.clipIter)
409  statsCtrl.setAndMask(self.getBadPixelMask())
410  statsCtrl.setNanSafe(True)
411 
412  # compute tempExpRefList: a list of tempExpRef that actually exist
413  # and weightList: a list of the weight of the associated coadd tempExp
414  # and imageScalerList: a list of scale factors for the associated coadd tempExp
415  tempExpRefList = []
416  weightList = []
417  imageScalerList = []
418  tempExpName = self.getTempExpDatasetName()
419  for tempExpRef in refList:
420  if not tempExpRef.datasetExists(tempExpName):
421  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
422  continue
423 
424  tempExp = tempExpRef.get(tempExpName, immediate=True)
425  maskedImage = tempExp.getMaskedImage()
426  imageScaler = self.scaleZeroPoint.computeImageScaler(
427  exposure=tempExp,
428  dataRef=tempExpRef,
429  )
430  try:
431  imageScaler.scaleMaskedImage(maskedImage)
432  except Exception as e:
433  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
434  continue
435  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
436  afwMath.MEANCLIP, statsCtrl)
437  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
438  weight = 1.0 / float(meanVar)
439  if not numpy.isfinite(weight):
440  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
441  continue
442  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
443 
444  del maskedImage
445  del tempExp
446 
447  tempExpRefList.append(tempExpRef)
448  weightList.append(weight)
449  imageScalerList.append(imageScaler)
450 
451  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
452  imageScalerList=imageScalerList)
453 
454  def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None):
455  """!
456  \brief Perform background matching on the prepared inputs
457 
458  Each coaddTempExp has a different background level that must be normalized to a reference level
459  before coaddition. If no reference is provided, the background matcher selects one. If the background
460  matching is performed sucessfully, recompute the weight to be applied to the coaddTempExp to be
461  consistent with the scaled background.
462 
463  \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
464  \param[in] refExpDataRef: Data reference for background reference tempExp, or None
465  \param[in] refImageScaler: Image scaler for background reference tempExp, or None
466  \return Struct:
467  - tempExprefList: List of data references to tempExp
468  - weightList: List of weightings
469  - imageScalerList: List of image scalers
470  - backgroundInfoList: result from background matching
471  """
472  try:
473  backgroundInfoList = self.matchBackgrounds.run(
474  expRefList=inputData.tempExpRefList,
475  imageScalerList=inputData.imageScalerList,
476  refExpDataRef=refExpDataRef if not self.config.autoReference else None,
477  refImageScaler=refImageScaler,
478  expDatasetType=self.getTempExpDatasetName(),
479  ).backgroundInfoList
480  except Exception as e:
481  self.log.fatal("Cannot match backgrounds: %s", e)
482  raise pipeBase.TaskError("Background matching failed.")
483 
484  newWeightList = []
485  newTempExpRefList = []
486  newBackgroundStructList = []
487  newScaleList = []
488  # the number of good backgrounds may be < than len(tempExpList)
489  # sync these up and correct the weights
490  for tempExpRef, bgInfo, scaler, weight in zip(inputData.tempExpRefList, backgroundInfoList,
491  inputData.imageScalerList, inputData.weightList):
492  if not bgInfo.isReference:
493  # skip exposure if it has no backgroundModel
494  # or if fit was bad
495  if (bgInfo.backgroundModel is None):
496  self.log.info("No background offset model available for %s: skipping", tempExpRef.dataId)
497  continue
498  try:
499  varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
500  except Exception as e:
501  self.log.info("MSE/Var ratio not calculable (%s) for %s: skipping",
502  e, tempExpRef.dataId)
503  continue
504  if not numpy.isfinite(varianceRatio):
505  self.log.info("MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
506  bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
507  continue
508  elif (varianceRatio > self.config.maxMatchResidualRatio):
509  self.log.info("Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
510  varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
511  continue
512  elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
513  self.log.info("Bad fit. RMS %.2f > %.2f for %s: skipping",
514  bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
515  continue
516  newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
517  newTempExpRefList.append(tempExpRef)
518  newBackgroundStructList.append(bgInfo)
519  newScaleList.append(scaler)
520 
521  return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
522  imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
523 
524  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
525  altMaskList=None, doClip=False, mask=None):
526  """!
527  \anchor AssembleCoaddTask.assemble_
528 
529  \brief Assemble a coadd from input warps
530 
531  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
532  large area), the assembly is performed over small areas on the image at a time in order to
533  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
534  \ref assembleSubregion to mean-stack the corresponding subregions from the coaddTempExps (with outlier
535  rejection if config.doSigmaClip=True). Set the edge bits the the coadd mask based on the weight map.
536 
537  \param[in] skyInfo: Patch geometry information, from getSkyInfo
538  \param[in] tempExpRefList: List of data references to tempExp
539  \param[in] imageScalerList: List of image scalers
540  \param[in] weightList: List of weights
541  \param[in] bgInfoList: List of background data from background matching, or None
542  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
543  \param[in] doClip: Use clipping when codding?
544  \param[in] mask: Mask to ignore when coadding
545  \return coadded exposure
546  """
547  tempExpName = self.getTempExpDatasetName()
548  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
549  if mask is None:
550  mask = self.getBadPixelMask()
551 
552  statsCtrl = afwMath.StatisticsControl()
553  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
554  statsCtrl.setNumIter(self.config.clipIter)
555  statsCtrl.setAndMask(mask)
556  statsCtrl.setNanSafe(True)
557  statsCtrl.setWeighted(True)
558  statsCtrl.setCalcErrorFromInputVariance(True)
559  for plane, threshold in self.config.maskPropagationThresholds.items():
560  bit = afwImage.MaskU.getMaskPlane(plane)
561  statsCtrl.setMaskPropagationThreshold(bit, threshold)
562 
563  if doClip:
564  statsFlags = afwMath.MEANCLIP
565  else:
566  statsFlags = afwMath.MEAN
567 
568  if bgInfoList is None:
569  bgInfoList = [None]*len(tempExpRefList)
570 
571  if altMaskList is None:
572  altMaskList = [None]*len(tempExpRefList)
573 
574  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
575  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
576  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
577  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
578  coaddMaskedImage = coaddExposure.getMaskedImage()
579  subregionSizeArr = self.config.subregionSize
580  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
581  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
582  try:
583  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
584  weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
585  except Exception as e:
586  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
587 
588  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
589 
590  return coaddExposure
591 
592  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
593  """!
594  \brief Set the metadata for the coadd
595 
596  This basic implementation simply sets the filter from the
597  first input.
598 
599  \param[in] coaddExposure: The target image for the coadd
600  \param[in] tempExpRefList: List of data references to tempExp
601  \param[in] weightList: List of weights
602  """
603  assert len(tempExpRefList) == len(weightList), "Length mismatch"
604  tempExpName = self.getTempExpDatasetName()
605  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
606  # (and we need more than just the PropertySet that contains the header), which is not possible
607  # with the current butler (see #2777).
608  tempExpList = [tempExpRef.get(tempExpName + "_sub",
609  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
610  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
611  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
612 
613  coaddExposure.setFilter(tempExpList[0].getFilter())
614  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
615  coaddInputs.ccds.reserve(numCcds)
616  coaddInputs.visits.reserve(len(tempExpList))
617 
618  for tempExp, weight in zip(tempExpList, weightList):
619  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
620  coaddInputs.visits.sort()
621  if self.config.doPsfMatch:
622  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
623  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
624  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
625  # having the maximum width (sufficient because square)
626  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
627  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
628  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
629  else:
630  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
631  coaddExposure.setPsf(psf)
632  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
633  coaddExposure.getWcs())
634  coaddExposure.getInfo().setApCorrMap(apCorrMap)
635 
636  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
637  bgInfoList, altMaskList, statsFlags, statsCtrl):
638  """!
639  \brief Assemble the coadd for a sub-region.
640 
641  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
642  matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
643  planes listed in config.removeMaskPlanes, Finally, mean-stack
644  the actual exposures using \ref afwMath.statisticsStack "statisticsStack" with outlier rejection if
645  config.doSigmaClip=True. Assign the stacked subregion back to the coadd.
646 
647  \param[in] coaddExposure: The target image for the coadd
648  \param[in] bbox: Sub-region to coadd
649  \param[in] tempExpRefList: List of data reference to tempExp
650  \param[in] imageScalerList: List of image scalers
651  \param[in] weightList: List of weights
652  \param[in] bgInfoList: List of background data from background matching
653  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
654  \param[in] statsFlags: Statistic for coadd
655  \param[in] statsCtrl: Statistics control object for coadd
656  """
657  self.log.debug("Computing coadd over %s", bbox)
658  tempExpName = self.getTempExpDatasetName()
659  coaddMaskedImage = coaddExposure.getMaskedImage()
660  maskedImageList = []
661  for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
662  altMaskList):
663  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
664  maskedImage = exposure.getMaskedImage()
665 
666  if altMask:
667  altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
668  maskedImage.getMask().swap(altMaskSub)
669  imageScaler.scaleMaskedImage(maskedImage)
670 
671  if self.config.doMatchBackgrounds and not bgInfo.isReference:
672  backgroundModel = bgInfo.backgroundModel
673  backgroundImage = backgroundModel.getImage() if \
674  self.matchBackgrounds.config.usePolynomial else \
675  backgroundModel.getImageF()
676  backgroundImage.setXY0(coaddMaskedImage.getXY0())
677  maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
678  var = maskedImage.getVariance()
679  var += (bgInfo.fitRMS)**2
680 
681  if self.config.removeMaskPlanes:
682  mask = maskedImage.getMask()
683  for maskPlane in self.config.removeMaskPlanes:
684  try:
685  mask &= ~mask.getPlaneBitMask(maskPlane)
686  except Exception as e:
687  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.message)
688 
689  maskedImageList.append(maskedImage)
690 
691  with self.timer("stack"):
692  coaddSubregion = afwMath.statisticsStack(
693  maskedImageList, statsFlags, statsCtrl, weightList)
694 
695  coaddMaskedImage.assign(coaddSubregion, bbox)
696 
697  def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList):
698  """!
699  \brief Add metadata from the background matching to the coadd
700 
701  \param[in] coaddExposure: Coadd
702  \param[in] tempExpRefList: List of data references for temp exps to go into coadd
703  \param[in] backgroundInfoList: List of background info, results from background matching
704  """
705  self.log.info("Adding exposure information to metadata")
706  metadata = coaddExposure.getMetadata()
707  metadata.addString("CTExp_SDQA1_DESCRIPTION",
708  "Background matching: Ratio of matchedMSE / diffImVar")
709  for ind, (tempExpRef, backgroundInfo) in enumerate(zip(tempExpRefList, backgroundInfoList)):
710  tempExpStr = '&'.join('%s=%s' % (k, v) for k, v in tempExpRef.dataId.items())
711  if backgroundInfo.isReference:
712  metadata.addString("ReferenceExp_ID", tempExpStr)
713  else:
714  metadata.addString("CTExp_ID_%d" % (ind), tempExpStr)
715  metadata.addDouble("CTExp_SDQA1_%d" % (ind),
716  backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
717  metadata.addDouble("CTExp_SDQA2_%d" % (ind),
718  backgroundInfo.fitRMS)
719 
720  def readBrightObjectMasks(self, dataRef):
721  """Returns None on failure"""
722  try:
723  return dataRef.get("brightObjectMask", immediate=True)
724  except Exception as e:
725  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
726  return None
727 
728  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
729  """Set the bright object masks
730 
731  exposure: Exposure under consideration
732  dataId: Data identifier dict for patch
733  brightObjectMasks: afwTable of bright objects to mask
734  """
735  #
736  # Check the metadata specifying the tract/patch/filter
737  #
738  if brightObjectMasks is None:
739  self.log.warn("Unable to apply bright object mask: none supplied")
740  return
741  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
742  md = brightObjectMasks.table.getMetadata()
743  for k in dataId:
744  if not md.exists(k):
745  self.log.warn("Expected to see %s in metadata", k)
746  else:
747  if md.get(k) != dataId[k]:
748  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
749 
750  mask = exposure.getMaskedImage().getMask()
751  wcs = exposure.getWcs()
752  plateScale = wcs.pixelScale().asArcseconds()
753 
754  for rec in brightObjectMasks:
755  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
756  if rec["type"] == "box":
757  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
758  width = rec["width"].asArcseconds()/plateScale # convert to pixels
759  height = rec["height"].asArcseconds()/plateScale # convert to pixels
760 
761  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
762  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
763 
764  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
765  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
766  foot = afwDetect.Footprint(bbox, exposure.getBBox())
767  elif rec["type"] == "circle":
768  radius = rec["radius"].asArcseconds()/plateScale # convert to pixels
769  foot = afwDetect.Footprint(center, radius, exposure.getBBox())
770  else:
771  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
772  continue
773 
774  afwDetect.setMaskFromFootprint(mask, foot, self.brightObjectBitmask)
775 
776  @classmethod
777  def _makeArgumentParser(cls):
778  """!
779  \brief Create an argument parser
780  """
781  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
782  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_tempExp",
783  help="data ID, e.g. --id tract=12345 patch=1,2",
784  ContainerClass=AssembleCoaddDataIdContainer)
785  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
786  ContainerClass=SelectDataIdContainer)
787  return parser
788 
789 
790 def _subBBoxIter(bbox, subregionSize):
791  """!
792  \brief Iterate over subregions of a bbox
793 
794  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
795  \param[in] subregionSize: size of sub-bboxes
796 
797  \return subBBox: next sub-bounding box of size subregionSize or smaller;
798  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
799  but it will never be empty
800  """
801  if bbox.isEmpty():
802  raise RuntimeError("bbox %s is empty" % (bbox,))
803  if subregionSize[0] < 1 or subregionSize[1] < 1:
804  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
805 
806  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
807  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
808  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
809  subBBox.clip(bbox)
810  if subBBox.isEmpty():
811  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
812  (bbox, subregionSize, colShift, rowShift))
813  yield subBBox
814 
815 
816 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
817  """!
818  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
819  """
820 
821  def makeDataRefList(self, namespace):
822  """!
823  \brief Make self.refList from self.idList.
824 
825  Interpret the config.doMatchBackgrounds, config.autoReference,
826  and whether a visit/run supplied.
827  If a visit/run is supplied, config.autoReference is automatically set to False.
828  if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
829 
830  """
831  keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName + "Coadd",
832  level=self.level)
833  keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName + "Coadd_tempExp",
834  level=self.level)
835 
836  if namespace.config.doMatchBackgrounds:
837  if namespace.config.autoReference: # matcher will pick it's own reference image
838  datasetType = namespace.config.coaddName + "Coadd"
839  validKeys = keysCoadd
840  else:
841  datasetType = namespace.config.coaddName + "Coadd_tempExp"
842  validKeys = keysCoaddTempExp
843  else: # bkg subtracted coadd
844  datasetType = namespace.config.coaddName + "Coadd"
845  validKeys = keysCoadd
846 
847  for dataId in self.idList:
848  # tract and patch are required
849  for key in validKeys:
850  if key not in dataId:
851  raise RuntimeError("--id must include " + key)
852 
853  for key in dataId: # check if users supplied visit/run
854  if (key not in keysCoadd) and (key in keysCoaddTempExp): # user supplied a visit/run
855  if namespace.config.autoReference:
856  # user probably meant: autoReference = False
857  namespace.config.autoReference = False
858  datasetType = namespace.config.coaddName + "Coadd_tempExp"
859  print("Switching config.autoReference to False; applies only to background Matching.")
860  break
861 
862  dataRef = namespace.butler.dataRef(
863  datasetType=datasetType,
864  dataId=dataId,
865  )
866  self.refList.append(dataRef)
867 
868 
869 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
870  """!
871  \brief Function to count the number of pixels with a specific mask in a footprint.
872 
873  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
874  have bitmask set but do not have ignoreMask set. Return the count.
875 
876  \param[in] mask: mask to define intersection region by.
877  \parma[in] footprint: footprint to define the intersection region by.
878  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
879  \param[in] ignoreMask: pixels to not consider.
880  \return count of number of pixels in footprint with specified mask.
881  """
882  bbox = footprint.getBBox()
883  bbox.clip(mask.getBBox(afwImage.PARENT))
884  fp = afwImage.MaskU(bbox)
885  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
886  footprint.spans.setMask(fp, bitmask)
887  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
888  (subMask.getArray() & ignoreMask) == 0).sum()
889 
890 
892  """!
893 \anchor SafeClipAssembleCoaddConfig
894 
895 \brief Configuration parameters for the SafeClipAssembleCoaddTask
896  """
897  clipDetection = pexConfig.ConfigurableField(
898  target=SourceDetectionTask,
899  doc="Detect sources on difference between unclipped and clipped coadd")
900  minClipFootOverlap = pexConfig.Field(
901  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
902  dtype=float,
903  default=0.6
904  )
905  minClipFootOverlapSingle = pexConfig.Field(
906  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
907  "clipped when only one visit overlaps",
908  dtype=float,
909  default=0.5
910  )
911  minClipFootOverlapDouble = pexConfig.Field(
912  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
913  "clipped when two visits overlap",
914  dtype=float,
915  default=0.45
916  )
917  maxClipFootOverlapDouble = pexConfig.Field(
918  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
919  "considering two visits",
920  dtype=float,
921  default=0.15
922  )
923  minBigOverlap = pexConfig.Field(
924  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
925  "when labeling clipped footprints",
926  dtype=int,
927  default=100
928  )
929 
930  def setDefaults(self):
931  # The numeric values for these configuration parameters were empirically determined, future work
932  # may further refine them.
933  AssembleCoaddConfig.setDefaults(self)
934  self.clipDetection.doTempLocalBackground = False
935  self.clipDetection.reEstimateBackground = False
936  self.clipDetection.returnOriginalFootprints = False
937  self.clipDetection.thresholdPolarity = "both"
938  self.clipDetection.thresholdValue = 2
939  self.clipDetection.nSigmaToGrow = 2
940  self.clipDetection.minPixels = 4
941  self.clipDetection.isotropicGrow = True
942  self.clipDetection.thresholdType = "pixel_stdev"
943  self.sigmaClip = 1.5
944  self.clipIter = 3
945 
946 ## \addtogroup LSST_task_documentation
947 ## \{
948 ## \page SafeClipAssembleCoaddTask
949 ## \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask"
950 ## \copybrief SafeClipAssembleCoaddTask
951 ## \}
952 
953 
955  """!
956  \anchor SafeClipAssembleCoaddTask_
957 
958  \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
959  with potential artifacts.
960 
961  \section pipe_tasks_assembleCoadd_Contents Contents
962  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
963  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
964  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
965  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
966  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
967  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
968 
969  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
970 
971  \copybrief SafeClipAssembleCoaddTask
972 
973  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
974  SafeClipAssembleCoaddTask subtasks that task.
975  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
976  outliers).
977  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
978  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
979  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
980  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
981  that there is an outlier and ii. this outlier appears on only one or two images.
982  Such regions will not contribute to the final coadd.
983  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
984  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
985  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
986  surveys.
987 
988  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
989  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
990  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
991 
992  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
993  \copydoc \_\_init\_\_
994 
995  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
996  \copydoc run
997 
998  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
999  See \ref SafeClipAssembleCoaddConfig
1000 
1001  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1002  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1003  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1004  files.
1005  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1006  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1007  for further information.
1008 
1009  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1010 
1011  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1012  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1013  '--legacyCoadd'.
1014  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1015  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1016  with a list of coaddTempExps to attempt to coadd (specified using
1017  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1018  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1019  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1020  command line argument:
1021  \code
1022  assembleCoadd.py --help
1023  \endcode
1024  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1025  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1026  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1027  packages.
1028  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1029  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1030  <DL>
1031  <DT>processCcd</DT>
1032  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1033  <DT>makeSkyMap</DT>
1034  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1035  <DT>makeCoaddTempExp</DT>
1036  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1037  </DL>
1038  We can perform all of these steps by running
1039  \code
1040  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1041  \endcode
1042  This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1043  as follows:
1044  \code
1045  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1046  \endcode
1047  This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1048  You may also choose to run:
1049  \code
1050  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1051  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1052  \endcode
1053  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1054  discussed in \ref pipeTasks_multiBand.
1055  """
1056  ConfigClass = SafeClipAssembleCoaddConfig
1057  _DefaultName = "safeClipAssembleCoadd"
1058 
1059  def __init__(self, *args, **kwargs):
1060  """!
1061  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1062  """
1063  AssembleCoaddTask.__init__(self, *args, **kwargs)
1064  schema = afwTable.SourceTable.makeMinimalSchema()
1065  self.makeSubtask("clipDetection", schema=schema)
1066 
1067  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1068  """!
1069  \brief Assemble the coadd for a region
1070 
1071  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1072  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1073  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1074  overlap between the clipped region and a source.
1075  This leaves us with a set of footprints from the difference image that have been identified as having
1076  occured on just one or two individual visits. However, these footprints were generated from a
1077  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1078  footprints acrosss the coadd difference in this process.
1079  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1080  these are big footprints.
1081  Combine the small and big clipped footprints and mark them on a new bad mask plane
1082  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1083  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1084  bad mask plane.
1085 
1086  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1087  parent task.
1088 
1089  @param skyInfo: Patch geometry information, from getSkyInfo
1090  @param tempExpRefList: List of data reference to tempExp
1091  @param imageScalerList: List of image scalers
1092  @param weightList: List of weights
1093  @param bgModelList: List of background models from background matching
1094  return coadd exposure
1095  """
1096  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1097  mask = exp.getMaskedImage().getMask()
1098  mask.addMaskPlane("CLIPPED")
1099 
1100  result = self.detectClip(exp, tempExpRefList)
1101 
1102  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1103 
1104  # Go to individual visits for big footprints
1105  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1106  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1107  bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1108  maskClipValue, maskDetValue)
1109 
1110  # Create mask of the current clipped footprints
1111  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1112  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1113 
1114  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1115  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1116  maskClip |= maskClipBig
1117 
1118  # Assemble coadd from base class, but ignoring CLIPPED pixels (doClip is false)
1119  badMaskPlanes = self.config.badMaskPlanes[:]
1120  badMaskPlanes.append("CLIPPED")
1121  badPixelMask = afwImage.MaskU.getPlaneBitMask(badMaskPlanes)
1122  coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1123  bgModelList, result.tempExpClipList,
1124  doClip=False,
1125  mask=badPixelMask)
1126 
1127  # Set the coadd CLIPPED mask from the footprints since currently pixels that are masked
1128  # do not get propagated
1129  maskExp = coaddExp.getMaskedImage().getMask()
1130  maskExp |= maskClip
1131 
1132  return coaddExp
1133 
1134  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList):
1135  """!
1136  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1137 
1138  Generate a difference image between clipped and unclipped coadds.
1139  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1140  Return the difference image.
1141 
1142  @param skyInfo: Patch geometry information, from getSkyInfo
1143  @param tempExpRefList: List of data reference to tempExp
1144  @param imageScalerList: List of image scalers
1145  @param weightList: List of weights
1146  @param bgModelList: List of background models from background matching
1147  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1148  """
1149  # Build the unclipped coadd
1150  coaddMean = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1151  bgModelList, doClip=False)
1152 
1153  # Build the clipped coadd
1154  coaddClip = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1155  bgModelList, doClip=True)
1156 
1157  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1158  coaddDiff -= coaddClip.getMaskedImage()
1159  exp = afwImage.ExposureF(coaddDiff)
1160  exp.setPsf(coaddMean.getPsf())
1161  return exp
1162 
1163  def detectClip(self, exp, tempExpRefList):
1164  """!
1165  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1166 
1167  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1168  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1169  fraction to thresholds set in the config.
1170  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1171  two).
1172  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1173  the coaddTempExp.
1174  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1175  with the clipped footprints and a list of new masks for the coaddTempExps.
1176 
1177  \param[in] exp: Exposure to run detection on
1178  \param[in] tempExpRefList: List of data reference to tempExp
1179  \return struct containing:
1180  - clippedFootprints: list of clipped footprints
1181  - clippedIndices: indices for each clippedFootprint in tempExpRefList
1182  - tempExpClipList: list of new masks for tempExp
1183  """
1184  mask = exp.getMaskedImage().getMask()
1185  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1186  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1187  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1188  # Merge positive and negative together footprints together
1189  fpSet.positive.merge(fpSet.negative)
1190  footprints = fpSet.positive
1191  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1192  ignoreMask = self.getBadPixelMask()
1193 
1194  clipFootprints = []
1195  clipIndices = []
1196 
1197  # build a list with a mask for each visit which can be modified with clipping information
1198  tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(),
1199  immediate=True).getMaskedImage().getMask() for
1200  tmpExpRef in tempExpRefList]
1201 
1202  for footprint in footprints.getFootprints():
1203  nPixel = footprint.getArea()
1204  overlap = [] # hold the overlap with each visit
1205  maskList = [] # which visit mask match
1206  indexList = [] # index of visit in global list
1207  for i, tmpExpMask in enumerate(tempExpClipList):
1208  # Determine the overlap with the footprint
1209  ignore = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1210  overlapDet = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1211  totPixel = nPixel - ignore
1212 
1213  # If we have more bad pixels than detection skip
1214  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1215  continue
1216  overlap.append(overlapDet/float(totPixel))
1217  maskList.append(tmpExpMask)
1218  indexList.append(i)
1219 
1220  overlap = numpy.array(overlap)
1221  if not len(overlap):
1222  continue
1223 
1224  keep = False # Should this footprint be marked as clipped?
1225  keepIndex = [] # Which tempExps does the clipped footprint belong to
1226 
1227  # If footprint only has one overlap use a lower threshold
1228  if len(overlap) == 1:
1229  if overlap[0] > self.config.minClipFootOverlapSingle:
1230  keep = True
1231  keepIndex = [0]
1232  else:
1233  # This is the general case where only visit should be clipped
1234  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1235  if len(clipIndex) == 1:
1236  keep = True
1237  keepIndex = [clipIndex[0]]
1238 
1239  # Test if there are clipped objects that overlap two different visits
1240  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1241  if len(clipIndex) == 2 and len(overlap) > 3:
1242  clipIndexComp = numpy.where(overlap < self.config.minClipFootOverlapDouble)[0]
1243  if numpy.max(overlap[clipIndexComp]) < self.config.maxClipFootOverlapDouble:
1244  keep = True
1245  keepIndex = clipIndex
1246 
1247  if not keep:
1248  continue
1249 
1250  for index in keepIndex:
1251  footprint.spans.setMask(maskList[index], maskClipValue)
1252 
1253  clipIndices.append(numpy.array(indexList)[keepIndex])
1254  clipFootprints.append(footprint)
1255 
1256  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1257  tempExpClipList=tempExpClipList)
1258 
1259  def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1260  """!
1261  \brief Find footprints from individual tempExp footprints for large footprints.
1262 
1263  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1264  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1265  significantly with each source in all the coaddTempExps.
1266 
1267  \param[in] tempExpClipList: List of tempExp masks with clipping information
1268  \param[in] clipFootprints: List of clipped footprints
1269  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1270  \param[in] maskClipValue: Mask value of clipped pixels
1271  \param[in] maskClipValue: Mask value of detected pixels
1272  \return list of big footprints
1273  """
1274  bigFootprintsCoadd = []
1275  ignoreMask = self.getBadPixelMask()
1276  for index, tmpExpMask in enumerate(tempExpClipList):
1277 
1278  # Create list of footprints from the DETECTED pixels
1279  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1280  afwImage.PARENT, True)
1281  maskVisitDet &= maskDetValue
1282  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1283 
1284  # build a mask of clipped footprints that are in this visit
1285  clippedFootprintsVisit = []
1286  for foot, clipIndex in zip(clipFootprints, clipIndices):
1287  if index not in clipIndex:
1288  continue
1289  clippedFootprintsVisit.append(foot)
1290  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1291  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1292 
1293  bigFootprintsVisit = []
1294  for foot in visitFootprints.getFootprints():
1295  if foot.getArea() < self.config.minBigOverlap:
1296  continue
1297  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1298  if nCount > self.config.minBigOverlap:
1299  bigFootprintsVisit.append(foot)
1300  bigFootprintsCoadd.append(foot)
1301 
1302  # Update single visit masks
1303  maskVisitClip.clearAllMaskPlanes()
1304  afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1305  tmpExpMask |= maskVisitClip
1306 
1307  return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of coaddTempExp.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of coadded temporary exposures.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip &amp; flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list of coaddTempExp data references corresponding to exposures that lie within the patch to...