lsst.pipe.tasks  13.0-37-g58c8d4e+4
 All Classes Namespaces Files Functions Variables Groups Pages
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import numpy
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.afw.detection as afwDetect
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage import InterpImageTask
39 from .matchBackgrounds import MatchBackgroundsTask
40 from .scaleZeroPoint import ScaleZeroPointTask
41 from .coaddHelpers import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms import SourceDetectionTask
43 
44 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask"]
45 
46 
47 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
48  """!
49 \anchor AssembleCoaddConfig_
50 
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
52  """
53  subregionSize = pexConfig.ListField(
54  dtype=int,
55  doc="Width, height of stack subregion size; "
56  "make small enough that a full stack of images will fit into memory at once.",
57  length=2,
58  default=(2000, 2000),
59  )
60  doSigmaClip = pexConfig.Field(
61  dtype=bool,
62  doc="Perform sigma clipped outlier rejection? If False then compute a simple mean.",
63  default=True,
64  )
65  sigmaClip = pexConfig.Field(
66  dtype=float,
67  doc="Sigma for outlier rejection; ignored if doSigmaClip false.",
68  default=3.0,
69  )
70  clipIter = pexConfig.Field(
71  dtype=int,
72  doc="Number of iterations of outlier rejection; ignored if doSigmaClip false.",
73  default=2,
74  )
75  scaleZeroPoint = pexConfig.ConfigurableField(
76  target=ScaleZeroPointTask,
77  doc="Task to adjust the photometric zero point of the coadd temp exposures",
78  )
79  doInterp = pexConfig.Field(
80  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
81  dtype=bool,
82  default=True,
83  )
84  interpImage = pexConfig.ConfigurableField(
85  target=InterpImageTask,
86  doc="Task to interpolate (and extrapolate) over NaN pixels",
87  )
88  matchBackgrounds = pexConfig.ConfigurableField(
89  target=MatchBackgroundsTask,
90  doc="Task to match backgrounds",
91  )
92  maxMatchResidualRatio = pexConfig.Field(
93  doc="Maximum ratio of the mean squared error of the background matching model to the variance "
94  "of the difference in backgrounds",
95  dtype=float,
96  default=1.1
97  )
98  maxMatchResidualRMS = pexConfig.Field(
99  doc="Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
100  dtype=float,
101  default=1.0
102  )
103  doWrite = pexConfig.Field(
104  doc="Persist coadd?",
105  dtype=bool,
106  default=True,
107  )
108  doMatchBackgrounds = pexConfig.Field(
109  doc="Match backgrounds of coadd temp exposures before coadding them? "
110  "If False, the coadd temp expsosures must already have been background subtracted or matched",
111  dtype=bool,
112  default=True,
113  )
114  autoReference = pexConfig.Field(
115  doc="Automatically select the coadd temp exposure to use as a reference for background matching? "
116  "Ignored if doMatchBackgrounds false. "
117  "If False you must specify the reference temp exposure as the data Id",
118  dtype=bool,
119  default=True,
120  )
121  maskPropagationThresholds = pexConfig.DictField(
122  keytype=str,
123  itemtype=float,
124  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
125  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
126  "would have contributed exceeds this value."),
127  default={"SAT": 0.1},
128  )
129  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["CROSSTALK", "NOT_DEBLENDED"],
130  doc="Mask planes to remove before coadding")
131  #
132  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
133  # To make this useful you *must* also configure the flags.pixel algorithm, for example
134  # by adding
135  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
136  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
137  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
138  #
139  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
140  doc="Set mask and flag bits for bright objects?")
141  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
142  doc="Name of mask bit used for bright objects")
143 
144  def setDefaults(self):
145  CoaddBaseTask.ConfigClass.setDefaults(self)
146  self.badMaskPlanes = ["NO_DATA", "BAD", "CR", ]
147 
148  def validate(self):
149  CoaddBaseTask.ConfigClass.validate(self)
150  if self.makeDirect and self.makePsfMatched:
151  raise ValueError("Currently, assembleCoadd can only make either Direct or PsfMatched Coadds "
152  "at a time. Set either makeDirect or makePsfMatched to False")
153 
154 
155 # \addtogroup LSST_task_documentation
156 # \{
157 # \page AssembleCoaddTask
158 # \ref AssembleCoaddTask_ "AssembleCoaddTask"
159 # \copybrief AssembleCoaddTask
160 # \}
161 
162 class AssembleCoaddTask(CoaddBaseTask):
163  """!
164 \anchor AssembleCoaddTask_
165 
166 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
167 
168 \section pipe_tasks_assembleCoadd_Contents Contents
169  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
170  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
171  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
172  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
173  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
174  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
175 
176 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
177 
178 \copybrief AssembleCoaddTask_
179 
180 We want to assemble a coadded image from a set of Warps (also called
181 coadded temporary exposures or coaddTempExps.
182 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
183 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
184 Warps that cover the specified patch (pointed at by dataRef).
185 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
186 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
187 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
188 When background matching is enabled, the task may be configured to automatically select a reference exposure
189 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
190 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
191 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
192 config.makePsfMatched set which of the warp types will be coadded.
193 The coadd is computed as a mean with optional outlier rejection.
194 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
195 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
196 
197 AssembleCoaddTask uses several sub-tasks. These are
198 <DL>
199  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
200  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
201  <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
202  <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
203  not provided).</DD>
204  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
205  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
206 </DL>
207 You can retarget these subtasks if you wish.
208 
209 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
210 \copydoc \_\_init\_\_
211 
212 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
213 \copydoc run
214 
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
216 See \ref AssembleCoaddConfig_
217 
218 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
219 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
220 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
221 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
222 the documetation for the subtasks for further information.
223 
224 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
225 
226 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
227 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
228 a data reference to the tract patch and filter to be coadded (specified using
229 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
230 Warps to attempt to coadd (specified using
231 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
232 that cover the specified tract and patch will be coadded. A list of the available optional
233 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
234 \code
235 assembleCoadd.py --help
236 \endcode
237 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
238 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
239 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
240 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
241 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
242 <DL>
243  <DT>processCcd</DT>
244  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
245  <DT>makeSkyMap</DT>
246  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
247  <DT>makeCoaddTempExp</DT>
248  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
249 </DL>
250 We can perform all of these steps by running
251 \code
252 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
253 \endcode
254 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
255 follows:
256 \code
257 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
258 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
259 You may also choose to run:
260 \code
261 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
262 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
263 \endcode
264 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
265 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
266 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
267  """
268  ConfigClass = AssembleCoaddConfig
269  _DefaultName = "assembleCoadd"
270 
271  def __init__(self, *args, **kwargs):
272  """!
273  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
274  \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
275  """
276  CoaddBaseTask.__init__(self, *args, **kwargs)
277  self.makeSubtask("interpImage")
278  self.makeSubtask("matchBackgrounds")
279  self.makeSubtask("scaleZeroPoint")
280 
281  if self.config.doMaskBrightObjects:
282  mask = afwImage.MaskU()
283  try:
284  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
285  except pexExceptions.LsstCppException:
286  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
287  mask.getMaskPlaneDict().keys())
288  del mask
289 
290  if self.config.makeDirect:
291  self.warpType = "direct"
292  elif self.config.makePsfMatched:
293  self.warpType = "psfMatched"
294  else:
295  raise ValueError("Neither makeDirect nor makePsfMatched configs are True")
296 
297  @pipeBase.timeMethod
298  def run(self, dataRef, selectDataList=[]):
299  """!
300  \brief Assemble a coadd from a set of Warps
301 
302  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
303  match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
304  Warps if the background has not already been removed. Assemble the Warps using
305  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
306  exposure.
307 
308  \anchor runParams
309  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
310  (if config.autoReference=False). Used to access the following data products:
311  - [in] self.config.coaddName + "Coadd_skyMap"
312  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
313  - [out] self.config.coaddName + "Coadd"
314  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
315  selected from this list based on overlap with the patch defined by dataRef.
316 
317  \return a pipeBase.Struct with fields:
318  - coaddExposure: coadded exposure
319  """
320  skyInfo = self.getSkyInfo(dataRef)
321  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
322  if len(calExpRefList) == 0:
323  self.log.warn("No exposures to coadd")
324  return
325  self.log.info("Coadding %d exposures", len(calExpRefList))
326 
327  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
328  inputData = self.prepareInputs(tempExpRefList)
329  self.log.info("Found %d %s", len(inputData.tempExpRefList),
330  self.getTempExpDatasetName(self.warpType))
331  if len(inputData.tempExpRefList) == 0:
332  self.log.warn("No coadd temporary exposures found")
333  return
334  if self.config.doMatchBackgrounds:
335  refImageScaler = self.getBackgroundReferenceScaler(dataRef)
336  inputData = self.backgroundMatching(inputData, dataRef, refImageScaler)
337  if len(inputData.tempExpRefList) == 0:
338  self.log.warn("No valid background models")
339  return
340 
341  coaddExp = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
342  inputData.weightList,
343  inputData.backgroundInfoList if self.config.doMatchBackgrounds else None,
344  doClip=self.config.doSigmaClip)
345  if self.config.doMatchBackgrounds:
346  self.addBackgroundMatchingMetadata(coaddExp, inputData.tempExpRefList,
347  inputData.backgroundInfoList)
348 
349  if self.config.doInterp:
350  self.interpImage.run(coaddExp.getMaskedImage(), planeName="NO_DATA")
351  # The variance must be positive; work around for DM-3201.
352  varArray = coaddExp.getMaskedImage().getVariance().getArray()
353  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
354 
355  if self.config.doMaskBrightObjects:
356  brightObjectMasks = self.readBrightObjectMasks(dataRef)
357  self.setBrightObjectMasks(coaddExp, dataRef.dataId, brightObjectMasks)
358 
359  if self.config.doWrite:
360  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
361  dataRef.put(coaddExp, self.getCoaddDatasetName(self.warpType))
362 
363  return pipeBase.Struct(coaddExposure=coaddExp)
364 
365  def getTempExpRefList(self, patchRef, calExpRefList):
366  """!
367  \brief Generate list data references corresponding to warped exposures that lie within the
368  patch to be coadded.
369 
370  \param[in] patchRef: Data reference for patch
371  \param[in] calExpRefList: List of data references for input calexps
372  \return List of Warp/CoaddTempExp data references
373  """
374  butler = patchRef.getButler()
375  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
376  self.getTempExpDatasetName(self.warpType))
377  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
378  g, groupData.keys) for
379  g in groupData.groups.keys()]
380  return tempExpRefList
381 
382  def getBackgroundReferenceScaler(self, dataRef):
383  """!
384  \brief Construct an image scaler for the background reference frame
385 
386  Each Warp has a different background level. A reference background level must be chosen before
387  coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
388  this routine is a no-op and None is returned. Otherwise, use the
389  \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
390  reference image and return it.
391 
392  \param[in] dataRef: Data reference for the background reference frame, or None
393  \return image scaler, or None
394  """
395  if self.config.autoReference:
396  return None
397 
398  # We've been given the data reference
399  dataset = self.getTempExpDatasetName(self.warpType)
400  if not dataRef.datasetExists(dataset):
401  raise RuntimeError("Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
402 
403  refExposure = dataRef.get(self.getTempExpDatasetName(self.warpType), immediate=True)
404  refImageScaler = self.scaleZeroPoint.computeImageScaler(
405  exposure=refExposure,
406  dataRef=dataRef,
407  )
408  return refImageScaler
409 
410  def prepareInputs(self, refList):
411  """!
412  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
413  for the photometric zero point.
414 
415  Each Warp has its own photometric zeropoint and background variance. Before coadding these
416  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
417  weight for each Warp.
418 
419  \param[in] refList: List of data references to tempExp
420  \return Struct:
421  - tempExprefList: List of data references to tempExp
422  - weightList: List of weightings
423  - imageScalerList: List of image scalers
424  """
425  statsCtrl = afwMath.StatisticsControl()
426  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
427  statsCtrl.setNumIter(self.config.clipIter)
428  statsCtrl.setAndMask(self.getBadPixelMask())
429  statsCtrl.setNanSafe(True)
430 
431  # compute tempExpRefList: a list of tempExpRef that actually exist
432  # and weightList: a list of the weight of the associated coadd tempExp
433  # and imageScalerList: a list of scale factors for the associated coadd tempExp
434  tempExpRefList = []
435  weightList = []
436  imageScalerList = []
437  tempExpName = self.getTempExpDatasetName(self.warpType)
438  for tempExpRef in refList:
439  if not tempExpRef.datasetExists(tempExpName):
440  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
441  continue
442 
443  tempExp = tempExpRef.get(tempExpName, immediate=True)
444  maskedImage = tempExp.getMaskedImage()
445  imageScaler = self.scaleZeroPoint.computeImageScaler(
446  exposure=tempExp,
447  dataRef=tempExpRef,
448  )
449  try:
450  imageScaler.scaleMaskedImage(maskedImage)
451  except Exception as e:
452  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
453  continue
454  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
455  afwMath.MEANCLIP, statsCtrl)
456  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
457  weight = 1.0 / float(meanVar)
458  if not numpy.isfinite(weight):
459  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
460  continue
461  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
462 
463  del maskedImage
464  del tempExp
465 
466  tempExpRefList.append(tempExpRef)
467  weightList.append(weight)
468  imageScalerList.append(imageScaler)
469 
470  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
471  imageScalerList=imageScalerList)
472 
473  def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None):
474  """!
475  \brief Perform background matching on the prepared inputs
476 
477  Each Warp has a different background level that must be normalized to a reference level
478  before coaddition. If no reference is provided, the background matcher selects one. If the background
479  matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
480  consistent with the scaled background.
481 
482  \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
483  \param[in] refExpDataRef: Data reference for background reference Warp, or None
484  \param[in] refImageScaler: Image scaler for background reference Warp, or None
485  \return Struct:
486  - tempExprefList: List of data references to warped exposures (coaddTempExps)
487  - weightList: List of weightings
488  - imageScalerList: List of image scalers
489  - backgroundInfoList: result from background matching
490  """
491  try:
492  backgroundInfoList = self.matchBackgrounds.run(
493  expRefList=inputData.tempExpRefList,
494  imageScalerList=inputData.imageScalerList,
495  refExpDataRef=refExpDataRef if not self.config.autoReference else None,
496  refImageScaler=refImageScaler,
497  expDatasetType=self.getTempExpDatasetName(self.warpType),
498  ).backgroundInfoList
499  except Exception as e:
500  self.log.fatal("Cannot match backgrounds: %s", e)
501  raise pipeBase.TaskError("Background matching failed.")
502 
503  newWeightList = []
504  newTempExpRefList = []
505  newBackgroundStructList = []
506  newScaleList = []
507  # the number of good backgrounds may be < than len(tempExpList)
508  # sync these up and correct the weights
509  for tempExpRef, bgInfo, scaler, weight in zip(inputData.tempExpRefList, backgroundInfoList,
510  inputData.imageScalerList, inputData.weightList):
511  if not bgInfo.isReference:
512  # skip exposure if it has no backgroundModel
513  # or if fit was bad
514  if (bgInfo.backgroundModel is None):
515  self.log.info("No background offset model available for %s: skipping", tempExpRef.dataId)
516  continue
517  try:
518  varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
519  except Exception as e:
520  self.log.info("MSE/Var ratio not calculable (%s) for %s: skipping",
521  e, tempExpRef.dataId)
522  continue
523  if not numpy.isfinite(varianceRatio):
524  self.log.info("MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
525  bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
526  continue
527  elif (varianceRatio > self.config.maxMatchResidualRatio):
528  self.log.info("Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
529  varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
530  continue
531  elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
532  self.log.info("Bad fit. RMS %.2f > %.2f for %s: skipping",
533  bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
534  continue
535  newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
536  newTempExpRefList.append(tempExpRef)
537  newBackgroundStructList.append(bgInfo)
538  newScaleList.append(scaler)
539 
540  return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
541  imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
542 
543  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
544  altMaskList=None, doClip=False, mask=None):
545  """!
546  \anchor AssembleCoaddTask.assemble_
547 
548  \brief Assemble a coadd from input warps
549 
550  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
551  large area), the assembly is performed over small areas on the image at a time in order to
552  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
553  \ref assembleSubregion to mean-stack the corresponding subregions from the coaddTempExps (with outlier
554  rejection if config.doSigmaClip=True). Set the edge bits the the coadd mask based on the weight map.
555 
556  \param[in] skyInfo: Patch geometry information, from getSkyInfo
557  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
558  \param[in] imageScalerList: List of image scalers
559  \param[in] weightList: List of weights
560  \param[in] bgInfoList: List of background data from background matching, or None
561  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
562  \param[in] doClip: Use clipping when codding?
563  \param[in] mask: Mask to ignore when coadding
564  \return coadded exposure
565  """
566  tempExpName = self.getTempExpDatasetName(self.warpType)
567  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
568  if mask is None:
569  mask = self.getBadPixelMask()
570 
571  statsCtrl = afwMath.StatisticsControl()
572  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
573  statsCtrl.setNumIter(self.config.clipIter)
574  statsCtrl.setAndMask(mask)
575  statsCtrl.setNanSafe(True)
576  statsCtrl.setWeighted(True)
577  statsCtrl.setCalcErrorFromInputVariance(True)
578  for plane, threshold in self.config.maskPropagationThresholds.items():
579  bit = afwImage.MaskU.getMaskPlane(plane)
580  statsCtrl.setMaskPropagationThreshold(bit, threshold)
581 
582  if doClip:
583  statsFlags = afwMath.MEANCLIP
584  else:
585  statsFlags = afwMath.MEAN
586 
587  if bgInfoList is None:
588  bgInfoList = [None]*len(tempExpRefList)
589 
590  if altMaskList is None:
591  altMaskList = [None]*len(tempExpRefList)
592 
593  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
594  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
595  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
596  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
597  coaddMaskedImage = coaddExposure.getMaskedImage()
598  subregionSizeArr = self.config.subregionSize
599  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
600  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
601  try:
602  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
603  weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
604  except Exception as e:
605  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
606 
607  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
608 
609  return coaddExposure
610 
611  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
612  """!
613  \brief Set the metadata for the coadd
614 
615  This basic implementation simply sets the filter from the
616  first input.
617 
618  \param[in] coaddExposure: The target image for the coadd
619  \param[in] tempExpRefList: List of data references to tempExp
620  \param[in] weightList: List of weights
621  """
622  assert len(tempExpRefList) == len(weightList), "Length mismatch"
623  tempExpName = self.getTempExpDatasetName(self.warpType)
624  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
625  # (and we need more than just the PropertySet that contains the header), which is not possible
626  # with the current butler (see #2777).
627  tempExpList = [tempExpRef.get(tempExpName + "_sub",
628  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
629  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
630  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
631 
632  coaddExposure.setFilter(tempExpList[0].getFilter())
633  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
634  coaddInputs.ccds.reserve(numCcds)
635  coaddInputs.visits.reserve(len(tempExpList))
636 
637  for tempExp, weight in zip(tempExpList, weightList):
638  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
639  coaddInputs.visits.sort()
640  if self.warpType == "psfMatched":
641  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
642  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
643  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
644  # having the maximum width (sufficient because square)
645  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
646  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
647  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
648  else:
649  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
650  coaddExposure.setPsf(psf)
651  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
652  coaddExposure.getWcs())
653  coaddExposure.getInfo().setApCorrMap(apCorrMap)
654 
655  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
656  bgInfoList, altMaskList, statsFlags, statsCtrl):
657  """!
658  \brief Assemble the coadd for a sub-region.
659 
660  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
661  matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
662  planes listed in config.removeMaskPlanes, Finally, mean-stack
663  the actual exposures using \ref afwMath.statisticsStack "statisticsStack" with outlier rejection if
664  config.doSigmaClip=True. Assign the stacked subregion back to the coadd.
665 
666  \param[in] coaddExposure: The target image for the coadd
667  \param[in] bbox: Sub-region to coadd
668  \param[in] tempExpRefList: List of data reference to tempExp
669  \param[in] imageScalerList: List of image scalers
670  \param[in] weightList: List of weights
671  \param[in] bgInfoList: List of background data from background matching
672  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
673  \param[in] statsFlags: Statistic for coadd
674  \param[in] statsCtrl: Statistics control object for coadd
675  """
676  self.log.debug("Computing coadd over %s", bbox)
677  tempExpName = self.getTempExpDatasetName(self.warpType)
678  coaddMaskedImage = coaddExposure.getMaskedImage()
679  maskedImageList = []
680  for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
681  altMaskList):
682  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
683  maskedImage = exposure.getMaskedImage()
684 
685  if altMask:
686  altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
687  maskedImage.getMask().swap(altMaskSub)
688  imageScaler.scaleMaskedImage(maskedImage)
689 
690  if self.config.doMatchBackgrounds and not bgInfo.isReference:
691  backgroundModel = bgInfo.backgroundModel
692  backgroundImage = backgroundModel.getImage() if \
693  self.matchBackgrounds.config.usePolynomial else \
694  backgroundModel.getImageF()
695  backgroundImage.setXY0(coaddMaskedImage.getXY0())
696  maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
697  var = maskedImage.getVariance()
698  var += (bgInfo.fitRMS)**2
699 
700  if self.config.removeMaskPlanes:
701  mask = maskedImage.getMask()
702  for maskPlane in self.config.removeMaskPlanes:
703  try:
704  mask &= ~mask.getPlaneBitMask(maskPlane)
705  except Exception as e:
706  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.message)
707 
708  maskedImageList.append(maskedImage)
709 
710  with self.timer("stack"):
711  coaddSubregion = afwMath.statisticsStack(
712  maskedImageList, statsFlags, statsCtrl, weightList)
713 
714  coaddMaskedImage.assign(coaddSubregion, bbox)
715 
716  def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList):
717  """!
718  \brief Add metadata from the background matching to the coadd
719 
720  \param[in] coaddExposure: Coadd
721  \param[in] tempExpRefList: List of data references for temp exps to go into coadd
722  \param[in] backgroundInfoList: List of background info, results from background matching
723  """
724  self.log.info("Adding exposure information to metadata")
725  metadata = coaddExposure.getMetadata()
726  metadata.addString("CTExp_SDQA1_DESCRIPTION",
727  "Background matching: Ratio of matchedMSE / diffImVar")
728  for ind, (tempExpRef, backgroundInfo) in enumerate(zip(tempExpRefList, backgroundInfoList)):
729  tempExpStr = '&'.join('%s=%s' % (k, v) for k, v in tempExpRef.dataId.items())
730  if backgroundInfo.isReference:
731  metadata.addString("ReferenceExp_ID", tempExpStr)
732  else:
733  metadata.addString("CTExp_ID_%d" % (ind), tempExpStr)
734  metadata.addDouble("CTExp_SDQA1_%d" % (ind),
735  backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
736  metadata.addDouble("CTExp_SDQA2_%d" % (ind),
737  backgroundInfo.fitRMS)
738 
739  def readBrightObjectMasks(self, dataRef):
740  """Returns None on failure"""
741  try:
742  return dataRef.get("brightObjectMask", immediate=True)
743  except Exception as e:
744  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
745  return None
746 
747  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
748  """Set the bright object masks
749 
750  exposure: Exposure under consideration
751  dataId: Data identifier dict for patch
752  brightObjectMasks: afwTable of bright objects to mask
753  """
754  #
755  # Check the metadata specifying the tract/patch/filter
756  #
757  if brightObjectMasks is None:
758  self.log.warn("Unable to apply bright object mask: none supplied")
759  return
760  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
761  md = brightObjectMasks.table.getMetadata()
762  for k in dataId:
763  if not md.exists(k):
764  self.log.warn("Expected to see %s in metadata", k)
765  else:
766  if md.get(k) != dataId[k]:
767  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
768 
769  mask = exposure.getMaskedImage().getMask()
770  wcs = exposure.getWcs()
771  plateScale = wcs.pixelScale().asArcseconds()
772 
773  for rec in brightObjectMasks:
774  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
775  if rec["type"] == "box":
776  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
777  width = rec["width"].asArcseconds()/plateScale # convert to pixels
778  height = rec["height"].asArcseconds()/plateScale # convert to pixels
779 
780  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
781  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
782 
783  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
784  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
785  spans = afwGeom.SpanSet(bbox)
786  elif rec["type"] == "circle":
787  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
788  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
789  else:
790  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
791  continue
792  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
793 
794  @classmethod
795  def _makeArgumentParser(cls):
796  """!
797  \brief Create an argument parser
798  """
799  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
800  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_directWarp",
801  help="data ID, e.g. --id tract=12345 patch=1,2",
802  ContainerClass=AssembleCoaddDataIdContainer)
803  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
804  ContainerClass=SelectDataIdContainer)
805  return parser
806 
807 
808 def _subBBoxIter(bbox, subregionSize):
809  """!
810  \brief Iterate over subregions of a bbox
811 
812  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
813  \param[in] subregionSize: size of sub-bboxes
814 
815  \return subBBox: next sub-bounding box of size subregionSize or smaller;
816  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
817  but it will never be empty
818  """
819  if bbox.isEmpty():
820  raise RuntimeError("bbox %s is empty" % (bbox,))
821  if subregionSize[0] < 1 or subregionSize[1] < 1:
822  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
823 
824  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
825  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
826  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
827  subBBox.clip(bbox)
828  if subBBox.isEmpty():
829  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
830  (bbox, subregionSize, colShift, rowShift))
831  yield subBBox
832 
833 
834 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
835  """!
836  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
837  """
838 
839  def makeDataRefList(self, namespace):
840  """!
841  \brief Make self.refList from self.idList.
842 
843  Interpret the config.doMatchBackgrounds, config.autoReference,
844  and whether a visit/run supplied.
845  If a visit/run is supplied, config.autoReference is automatically set to False.
846  if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
847 
848  """
849  keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName + "Coadd",
850  level=self.level)
851  keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
852  "Coadd_directWarp", level=self.level)
853 
854  if namespace.config.doMatchBackgrounds:
855  if namespace.config.autoReference: # matcher will pick it's own reference image
856  datasetType = namespace.config.coaddName + "Coadd"
857  validKeys = keysCoadd
858  else:
859  datasetType = namespace.config.coaddName + "Coadd_directWarp"
860  validKeys = keysCoaddTempExp
861  else: # bkg subtracted coadd
862  datasetType = namespace.config.coaddName + "Coadd"
863  validKeys = keysCoadd
864 
865  for dataId in self.idList:
866  # tract and patch are required
867  for key in validKeys:
868  if key not in dataId:
869  raise RuntimeError("--id must include " + key)
870 
871  for key in dataId: # check if users supplied visit/run
872  if (key not in keysCoadd) and (key in keysCoaddTempExp): # user supplied a visit/run
873  if namespace.config.autoReference:
874  # user probably meant: autoReference = False
875  namespace.config.autoReference = False
876  datasetType = namespace.config.coaddName + "Coadd_directWarp"
877  print("Switching config.autoReference to False; applies only to background Matching.")
878  break
879 
880  dataRef = namespace.butler.dataRef(
881  datasetType=datasetType,
882  dataId=dataId,
883  )
884  self.refList.append(dataRef)
885 
886 
887 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
888  """!
889  \brief Function to count the number of pixels with a specific mask in a footprint.
890 
891  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
892  have bitmask set but do not have ignoreMask set. Return the count.
893 
894  \param[in] mask: mask to define intersection region by.
895  \parma[in] footprint: footprint to define the intersection region by.
896  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
897  \param[in] ignoreMask: pixels to not consider.
898  \return count of number of pixels in footprint with specified mask.
899  """
900  bbox = footprint.getBBox()
901  bbox.clip(mask.getBBox(afwImage.PARENT))
902  fp = afwImage.MaskU(bbox)
903  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
904  footprint.spans.setMask(fp, bitmask)
905  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
906  (subMask.getArray() & ignoreMask) == 0).sum()
907 
908 
910  """!
911 \anchor SafeClipAssembleCoaddConfig
912 
913 \brief Configuration parameters for the SafeClipAssembleCoaddTask
914  """
915  clipDetection = pexConfig.ConfigurableField(
916  target=SourceDetectionTask,
917  doc="Detect sources on difference between unclipped and clipped coadd")
918  minClipFootOverlap = pexConfig.Field(
919  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
920  dtype=float,
921  default=0.6
922  )
923  minClipFootOverlapSingle = pexConfig.Field(
924  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
925  "clipped when only one visit overlaps",
926  dtype=float,
927  default=0.5
928  )
929  minClipFootOverlapDouble = pexConfig.Field(
930  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
931  "clipped when two visits overlap",
932  dtype=float,
933  default=0.45
934  )
935  maxClipFootOverlapDouble = pexConfig.Field(
936  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
937  "considering two visits",
938  dtype=float,
939  default=0.15
940  )
941  minBigOverlap = pexConfig.Field(
942  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
943  "when labeling clipped footprints",
944  dtype=int,
945  default=100
946  )
947 
948  def setDefaults(self):
949  # The numeric values for these configuration parameters were empirically determined, future work
950  # may further refine them.
951  AssembleCoaddConfig.setDefaults(self)
952  self.clipDetection.doTempLocalBackground = False
953  self.clipDetection.reEstimateBackground = False
954  self.clipDetection.returnOriginalFootprints = False
955  self.clipDetection.thresholdPolarity = "both"
956  self.clipDetection.thresholdValue = 2
957  self.clipDetection.nSigmaToGrow = 2
958  self.clipDetection.minPixels = 4
959  self.clipDetection.isotropicGrow = True
960  self.clipDetection.thresholdType = "pixel_stdev"
961  self.sigmaClip = 1.5
962  self.clipIter = 3
963 
964 ## \addtogroup LSST_task_documentation
965 ## \{
966 ## \page SafeClipAssembleCoaddTask
967 ## \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask"
968 ## \copybrief SafeClipAssembleCoaddTask
969 ## \}
970 
971 
973  """!
974  \anchor SafeClipAssembleCoaddTask_
975 
976  \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
977  with potential artifacts.
978 
979  \section pipe_tasks_assembleCoadd_Contents Contents
980  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
981  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
982  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
983  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
984  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
985  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
986 
987  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
988 
989  \copybrief SafeClipAssembleCoaddTask
990 
991  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
992  SafeClipAssembleCoaddTask subtasks that task.
993  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
994  outliers).
995  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
996  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
997  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
998  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
999  that there is an outlier and ii. this outlier appears on only one or two images.
1000  Such regions will not contribute to the final coadd.
1001  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1002  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1003  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1004  surveys.
1005 
1006  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1007  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1008  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1009 
1010  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1011  \copydoc \_\_init\_\_
1012 
1013  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1014  \copydoc run
1015 
1016  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1017  See \ref SafeClipAssembleCoaddConfig
1018 
1019  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1020  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1021  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1022  files.
1023  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1024  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1025  for further information.
1026 
1027  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1028 
1029  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1030  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1031  '--legacyCoadd'.
1032  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1033  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1034  with a list of coaddTempExps to attempt to coadd (specified using
1035  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1036  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1037  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1038  command line argument:
1039  \code
1040  assembleCoadd.py --help
1041  \endcode
1042  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1043  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1044  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1045  packages.
1046  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1047  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1048  <DL>
1049  <DT>processCcd</DT>
1050  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1051  <DT>makeSkyMap</DT>
1052  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1053  <DT>makeCoaddTempExp</DT>
1054  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1055  </DL>
1056  We can perform all of these steps by running
1057  \code
1058  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1059  \endcode
1060  This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1061  as follows:
1062  \code
1063  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1064  \endcode
1065  This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1066  You may also choose to run:
1067  \code
1068  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1069  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1070  \endcode
1071  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1072  discussed in \ref pipeTasks_multiBand.
1073  """
1074  ConfigClass = SafeClipAssembleCoaddConfig
1075  _DefaultName = "safeClipAssembleCoadd"
1076 
1077  def __init__(self, *args, **kwargs):
1078  """!
1079  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1080  """
1081  AssembleCoaddTask.__init__(self, *args, **kwargs)
1082  schema = afwTable.SourceTable.makeMinimalSchema()
1083  self.makeSubtask("clipDetection", schema=schema)
1084 
1085  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1086  """!
1087  \brief Assemble the coadd for a region
1088 
1089  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1090  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1091  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1092  overlap between the clipped region and a source.
1093  This leaves us with a set of footprints from the difference image that have been identified as having
1094  occured on just one or two individual visits. However, these footprints were generated from a
1095  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1096  footprints acrosss the coadd difference in this process.
1097  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1098  these are big footprints.
1099  Combine the small and big clipped footprints and mark them on a new bad mask plane
1100  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1101  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1102  bad mask plane.
1103 
1104  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1105  parent task.
1106 
1107  @param skyInfo: Patch geometry information, from getSkyInfo
1108  @param tempExpRefList: List of data reference to tempExp
1109  @param imageScalerList: List of image scalers
1110  @param weightList: List of weights
1111  @param bgModelList: List of background models from background matching
1112  return coadd exposure
1113  """
1114  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1115  mask = exp.getMaskedImage().getMask()
1116  mask.addMaskPlane("CLIPPED")
1117 
1118  result = self.detectClip(exp, tempExpRefList)
1119 
1120  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1121 
1122  # Go to individual visits for big footprints
1123  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1124  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1125  bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1126  maskClipValue, maskDetValue)
1127 
1128  # Create mask of the current clipped footprints
1129  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1130  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1131 
1132  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1133  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1134  maskClip |= maskClipBig
1135 
1136  # Assemble coadd from base class, but ignoring CLIPPED pixels (doClip is false)
1137  badMaskPlanes = self.config.badMaskPlanes[:]
1138  badMaskPlanes.append("CLIPPED")
1139  badPixelMask = afwImage.MaskU.getPlaneBitMask(badMaskPlanes)
1140  coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1141  bgModelList, result.tempExpClipList,
1142  doClip=False,
1143  mask=badPixelMask)
1144 
1145  # Set the coadd CLIPPED mask from the footprints since currently pixels that are masked
1146  # do not get propagated
1147  maskExp = coaddExp.getMaskedImage().getMask()
1148  maskExp |= maskClip
1149 
1150  return coaddExp
1151 
1152  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList):
1153  """!
1154  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1155 
1156  Generate a difference image between clipped and unclipped coadds.
1157  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1158  Return the difference image.
1159 
1160  @param skyInfo: Patch geometry information, from getSkyInfo
1161  @param tempExpRefList: List of data reference to tempExp
1162  @param imageScalerList: List of image scalers
1163  @param weightList: List of weights
1164  @param bgModelList: List of background models from background matching
1165  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1166  """
1167  # Build the unclipped coadd
1168  coaddMean = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1169  bgModelList, doClip=False)
1170 
1171  # Build the clipped coadd
1172  coaddClip = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1173  bgModelList, doClip=True)
1174 
1175  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1176  coaddDiff -= coaddClip.getMaskedImage()
1177  exp = afwImage.ExposureF(coaddDiff)
1178  exp.setPsf(coaddMean.getPsf())
1179  return exp
1180 
1181  def detectClip(self, exp, tempExpRefList):
1182  """!
1183  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1184 
1185  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1186  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1187  fraction to thresholds set in the config.
1188  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1189  two).
1190  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1191  the coaddTempExp.
1192  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1193  with the clipped footprints and a list of new masks for the coaddTempExps.
1194 
1195  \param[in] exp: Exposure to run detection on
1196  \param[in] tempExpRefList: List of data reference to tempExp
1197  \return struct containing:
1198  - clippedFootprints: list of clipped footprints
1199  - clippedIndices: indices for each clippedFootprint in tempExpRefList
1200  - tempExpClipList: list of new masks for tempExp
1201  """
1202  mask = exp.getMaskedImage().getMask()
1203  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1204  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1205  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1206  # Merge positive and negative together footprints together
1207  fpSet.positive.merge(fpSet.negative)
1208  footprints = fpSet.positive
1209  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1210  ignoreMask = self.getBadPixelMask()
1211 
1212  clipFootprints = []
1213  clipIndices = []
1214 
1215  # build a list with a mask for each visit which can be modified with clipping information
1216  tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.warpType),
1217  immediate=True).getMaskedImage().getMask() for
1218  tmpExpRef in tempExpRefList]
1219 
1220  for footprint in footprints.getFootprints():
1221  nPixel = footprint.getArea()
1222  overlap = [] # hold the overlap with each visit
1223  maskList = [] # which visit mask match
1224  indexList = [] # index of visit in global list
1225  for i, tmpExpMask in enumerate(tempExpClipList):
1226  # Determine the overlap with the footprint
1227  ignore = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1228  overlapDet = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1229  totPixel = nPixel - ignore
1230 
1231  # If we have more bad pixels than detection skip
1232  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1233  continue
1234  overlap.append(overlapDet/float(totPixel))
1235  maskList.append(tmpExpMask)
1236  indexList.append(i)
1237 
1238  overlap = numpy.array(overlap)
1239  if not len(overlap):
1240  continue
1241 
1242  keep = False # Should this footprint be marked as clipped?
1243  keepIndex = [] # Which tempExps does the clipped footprint belong to
1244 
1245  # If footprint only has one overlap use a lower threshold
1246  if len(overlap) == 1:
1247  if overlap[0] > self.config.minClipFootOverlapSingle:
1248  keep = True
1249  keepIndex = [0]
1250  else:
1251  # This is the general case where only visit should be clipped
1252  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1253  if len(clipIndex) == 1:
1254  keep = True
1255  keepIndex = [clipIndex[0]]
1256 
1257  # Test if there are clipped objects that overlap two different visits
1258  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1259  if len(clipIndex) == 2 and len(overlap) > 3:
1260  clipIndexComp = numpy.where(overlap < self.config.minClipFootOverlapDouble)[0]
1261  if numpy.max(overlap[clipIndexComp]) < self.config.maxClipFootOverlapDouble:
1262  keep = True
1263  keepIndex = clipIndex
1264 
1265  if not keep:
1266  continue
1267 
1268  for index in keepIndex:
1269  footprint.spans.setMask(maskList[index], maskClipValue)
1270 
1271  clipIndices.append(numpy.array(indexList)[keepIndex])
1272  clipFootprints.append(footprint)
1273 
1274  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1275  tempExpClipList=tempExpClipList)
1276 
1277  def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1278  """!
1279  \brief Find footprints from individual tempExp footprints for large footprints.
1280 
1281  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1282  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1283  significantly with each source in all the coaddTempExps.
1284 
1285  \param[in] tempExpClipList: List of tempExp masks with clipping information
1286  \param[in] clipFootprints: List of clipped footprints
1287  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1288  \param[in] maskClipValue: Mask value of clipped pixels
1289  \param[in] maskClipValue: Mask value of detected pixels
1290  \return list of big footprints
1291  """
1292  bigFootprintsCoadd = []
1293  ignoreMask = self.getBadPixelMask()
1294  for index, tmpExpMask in enumerate(tempExpClipList):
1295 
1296  # Create list of footprints from the DETECTED pixels
1297  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1298  afwImage.PARENT, True)
1299  maskVisitDet &= maskDetValue
1300  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1301 
1302  # build a mask of clipped footprints that are in this visit
1303  clippedFootprintsVisit = []
1304  for foot, clipIndex in zip(clipFootprints, clipIndices):
1305  if index not in clipIndex:
1306  continue
1307  clippedFootprintsVisit.append(foot)
1308  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1309  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1310 
1311  bigFootprintsVisit = []
1312  for foot in visitFootprints.getFootprints():
1313  if foot.getArea() < self.config.minBigOverlap:
1314  continue
1315  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1316  if nCount > self.config.minBigOverlap:
1317  bigFootprintsVisit.append(foot)
1318  bigFootprintsCoadd.append(foot)
1319 
1320  # Update single visit masks
1321  maskVisitClip.clearAllMaskPlanes()
1322  afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1323  tmpExpMask |= maskVisitClip
1324 
1325  return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of Warps.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip &amp; flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...