lsst.pipe.tasks  13.0-50-gf3eeffc+7
 All Classes Namespaces Files Functions Variables Groups Pages
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import numpy
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.afw.detection as afwDetect
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage import InterpImageTask
39 from .matchBackgrounds import MatchBackgroundsTask
40 from .scaleZeroPoint import ScaleZeroPointTask
41 from .coaddHelpers import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms import SourceDetectionTask
43 
44 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask"]
45 
46 
47 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
48  """!
49 \anchor AssembleCoaddConfig_
50 
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
52  """
53  subregionSize = pexConfig.ListField(
54  dtype=int,
55  doc="Width, height of stack subregion size; "
56  "make small enough that a full stack of images will fit into memory at once.",
57  length=2,
58  default=(2000, 2000),
59  )
60  doSigmaClip = pexConfig.Field(
61  dtype=bool,
62  doc="Perform sigma clipped outlier rejection? If False then compute a simple mean.",
63  default=True,
64  )
65  sigmaClip = pexConfig.Field(
66  dtype=float,
67  doc="Sigma for outlier rejection; ignored if doSigmaClip false.",
68  default=3.0,
69  )
70  clipIter = pexConfig.Field(
71  dtype=int,
72  doc="Number of iterations of outlier rejection; ignored if doSigmaClip false.",
73  default=2,
74  )
75  scaleZeroPoint = pexConfig.ConfigurableField(
76  target=ScaleZeroPointTask,
77  doc="Task to adjust the photometric zero point of the coadd temp exposures",
78  )
79  doInterp = pexConfig.Field(
80  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
81  dtype=bool,
82  default=True,
83  )
84  interpImage = pexConfig.ConfigurableField(
85  target=InterpImageTask,
86  doc="Task to interpolate (and extrapolate) over NaN pixels",
87  )
88  matchBackgrounds = pexConfig.ConfigurableField(
89  target=MatchBackgroundsTask,
90  doc="Task to match backgrounds",
91  )
92  maxMatchResidualRatio = pexConfig.Field(
93  doc="Maximum ratio of the mean squared error of the background matching model to the variance "
94  "of the difference in backgrounds",
95  dtype=float,
96  default=1.1
97  )
98  maxMatchResidualRMS = pexConfig.Field(
99  doc="Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
100  dtype=float,
101  default=1.0
102  )
103  doWrite = pexConfig.Field(
104  doc="Persist coadd?",
105  dtype=bool,
106  default=True,
107  )
108  doMatchBackgrounds = pexConfig.Field(
109  doc="Match backgrounds of coadd temp exposures before coadding them? "
110  "If False, the coadd temp expsosures must already have been background subtracted or matched",
111  dtype=bool,
112  default=True,
113  )
114  autoReference = pexConfig.Field(
115  doc="Automatically select the coadd temp exposure to use as a reference for background matching? "
116  "Ignored if doMatchBackgrounds false. "
117  "If False you must specify the reference temp exposure as the data Id",
118  dtype=bool,
119  default=True,
120  )
121  maskPropagationThresholds = pexConfig.DictField(
122  keytype=str,
123  itemtype=float,
124  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
125  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
126  "would have contributed exceeds this value."),
127  default={"SAT": 0.1},
128  )
129  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["CROSSTALK", "NOT_DEBLENDED"],
130  doc="Mask planes to remove before coadding")
131  #
132  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
133  # To make this useful you *must* also configure the flags.pixel algorithm, for example
134  # by adding
135  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
136  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
137  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
138  #
139  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
140  doc="Set mask and flag bits for bright objects?")
141  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
142  doc="Name of mask bit used for bright objects")
143 
144  coaddPsf = pexConfig.ConfigField(
145  doc="Configuration for CoaddPsf",
146  dtype=measAlg.CoaddPsfConfig,
147  )
148 
149  def setDefaults(self):
150  CoaddBaseTask.ConfigClass.setDefaults(self)
151  self.badMaskPlanes = ["NO_DATA", "BAD", "CR", ]
152 
153  def validate(self):
154  CoaddBaseTask.ConfigClass.validate(self)
155  if self.makeDirect and self.makePsfMatched:
156  raise ValueError("Currently, assembleCoadd can only make either Direct or PsfMatched Coadds "
157  "at a time. Set either makeDirect or makePsfMatched to False")
158 
159 
160 # \addtogroup LSST_task_documentation
161 # \{
162 # \page AssembleCoaddTask
163 # \ref AssembleCoaddTask_ "AssembleCoaddTask"
164 # \copybrief AssembleCoaddTask
165 # \}
166 
167 class AssembleCoaddTask(CoaddBaseTask):
168  """!
169 \anchor AssembleCoaddTask_
170 
171 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
172 
173 \section pipe_tasks_assembleCoadd_Contents Contents
174  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
175  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
176  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
177  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
178  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
179  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
180 
181 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
182 
183 \copybrief AssembleCoaddTask_
184 
185 We want to assemble a coadded image from a set of Warps (also called
186 coadded temporary exposures or coaddTempExps.
187 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
188 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
189 Warps that cover the specified patch (pointed at by dataRef).
190 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
191 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
192 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
193 When background matching is enabled, the task may be configured to automatically select a reference exposure
194 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
195 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
196 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
197 config.makePsfMatched set which of the warp types will be coadded.
198 The coadd is computed as a mean with optional outlier rejection.
199 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
200 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
201 
202 AssembleCoaddTask uses several sub-tasks. These are
203 <DL>
204  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
205  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
206  <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
207  <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
208  not provided).</DD>
209  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
210  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
211 </DL>
212 You can retarget these subtasks if you wish.
213 
214 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
215 \copydoc \_\_init\_\_
216 
217 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
218 \copydoc run
219 
220 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
221 See \ref AssembleCoaddConfig_
222 
223 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
224 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
225 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
226 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
227 the documetation for the subtasks for further information.
228 
229 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
230 
231 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
232 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
233 a data reference to the tract patch and filter to be coadded (specified using
234 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
235 Warps to attempt to coadd (specified using
236 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
237 that cover the specified tract and patch will be coadded. A list of the available optional
238 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
239 \code
240 assembleCoadd.py --help
241 \endcode
242 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
243 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
244 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
245 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
246 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
247 <DL>
248  <DT>processCcd</DT>
249  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
250  <DT>makeSkyMap</DT>
251  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
252  <DT>makeCoaddTempExp</DT>
253  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
254 </DL>
255 We can perform all of these steps by running
256 \code
257 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
258 \endcode
259 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
260 follows:
261 \code
262 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
263 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
264 You may also choose to run:
265 \code
266 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
267 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
268 \endcode
269 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
270 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
271 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
272  """
273  ConfigClass = AssembleCoaddConfig
274  _DefaultName = "assembleCoadd"
275 
276  def __init__(self, *args, **kwargs):
277  """!
278  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
279  \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
280  """
281  CoaddBaseTask.__init__(self, *args, **kwargs)
282  self.makeSubtask("interpImage")
283  self.makeSubtask("matchBackgrounds")
284  self.makeSubtask("scaleZeroPoint")
285 
286  if self.config.doMaskBrightObjects:
287  mask = afwImage.Mask()
288  try:
289  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
290  except pexExceptions.LsstCppException:
291  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
292  mask.getMaskPlaneDict().keys())
293  del mask
294 
295  if self.config.makeDirect:
296  self.warpType = "direct"
297  elif self.config.makePsfMatched:
298  self.warpType = "psfMatched"
299  else:
300  raise ValueError("Neither makeDirect nor makePsfMatched configs are True")
301 
302  @pipeBase.timeMethod
303  def run(self, dataRef, selectDataList=[]):
304  """!
305  \brief Assemble a coadd from a set of Warps
306 
307  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
308  match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
309  Warps if the background has not already been removed. Assemble the Warps using
310  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
311  exposure.
312 
313  \anchor runParams
314  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
315  (if config.autoReference=False). Used to access the following data products:
316  - [in] self.config.coaddName + "Coadd_skyMap"
317  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
318  - [out] self.config.coaddName + "Coadd"
319  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
320  selected from this list based on overlap with the patch defined by dataRef.
321 
322  \return a pipeBase.Struct with fields:
323  - coaddExposure: coadded exposure
324  """
325  skyInfo = self.getSkyInfo(dataRef)
326  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
327  if len(calExpRefList) == 0:
328  self.log.warn("No exposures to coadd")
329  return
330  self.log.info("Coadding %d exposures", len(calExpRefList))
331 
332  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
333  inputData = self.prepareInputs(tempExpRefList)
334  self.log.info("Found %d %s", len(inputData.tempExpRefList),
335  self.getTempExpDatasetName(self.warpType))
336  if len(inputData.tempExpRefList) == 0:
337  self.log.warn("No coadd temporary exposures found")
338  return
339  if self.config.doMatchBackgrounds:
340  refImageScaler = self.getBackgroundReferenceScaler(dataRef)
341  inputData = self.backgroundMatching(inputData, dataRef, refImageScaler)
342  if len(inputData.tempExpRefList) == 0:
343  self.log.warn("No valid background models")
344  return
345 
346  coaddExp = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
347  inputData.weightList,
348  inputData.backgroundInfoList if self.config.doMatchBackgrounds else None,
349  doClip=self.config.doSigmaClip)
350  if self.config.doMatchBackgrounds:
351  self.addBackgroundMatchingMetadata(coaddExp, inputData.tempExpRefList,
352  inputData.backgroundInfoList)
353 
354  if self.config.doInterp:
355  self.interpImage.run(coaddExp.getMaskedImage(), planeName="NO_DATA")
356  # The variance must be positive; work around for DM-3201.
357  varArray = coaddExp.getMaskedImage().getVariance().getArray()
358  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
359 
360  if self.config.doMaskBrightObjects:
361  brightObjectMasks = self.readBrightObjectMasks(dataRef)
362  self.setBrightObjectMasks(coaddExp, dataRef.dataId, brightObjectMasks)
363 
364  if self.config.doWrite:
365  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
366  dataRef.put(coaddExp, self.getCoaddDatasetName(self.warpType))
367 
368  return pipeBase.Struct(coaddExposure=coaddExp)
369 
370  def getTempExpRefList(self, patchRef, calExpRefList):
371  """!
372  \brief Generate list data references corresponding to warped exposures that lie within the
373  patch to be coadded.
374 
375  \param[in] patchRef: Data reference for patch
376  \param[in] calExpRefList: List of data references for input calexps
377  \return List of Warp/CoaddTempExp data references
378  """
379  butler = patchRef.getButler()
380  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
381  self.getTempExpDatasetName(self.warpType))
382  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
383  g, groupData.keys) for
384  g in groupData.groups.keys()]
385  return tempExpRefList
386 
387  def getBackgroundReferenceScaler(self, dataRef):
388  """!
389  \brief Construct an image scaler for the background reference frame
390 
391  Each Warp has a different background level. A reference background level must be chosen before
392  coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
393  this routine is a no-op and None is returned. Otherwise, use the
394  \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
395  reference image and return it.
396 
397  \param[in] dataRef: Data reference for the background reference frame, or None
398  \return image scaler, or None
399  """
400  if self.config.autoReference:
401  return None
402 
403  # We've been given the data reference
404  dataset = self.getTempExpDatasetName(self.warpType)
405  if not dataRef.datasetExists(dataset):
406  raise RuntimeError("Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
407 
408  refExposure = dataRef.get(self.getTempExpDatasetName(self.warpType), immediate=True)
409  refImageScaler = self.scaleZeroPoint.computeImageScaler(
410  exposure=refExposure,
411  dataRef=dataRef,
412  )
413  return refImageScaler
414 
415  def prepareInputs(self, refList):
416  """!
417  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
418  for the photometric zero point.
419 
420  Each Warp has its own photometric zeropoint and background variance. Before coadding these
421  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
422  weight for each Warp.
423 
424  \param[in] refList: List of data references to tempExp
425  \return Struct:
426  - tempExprefList: List of data references to tempExp
427  - weightList: List of weightings
428  - imageScalerList: List of image scalers
429  """
430  statsCtrl = afwMath.StatisticsControl()
431  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
432  statsCtrl.setNumIter(self.config.clipIter)
433  statsCtrl.setAndMask(self.getBadPixelMask())
434  statsCtrl.setNanSafe(True)
435 
436  # compute tempExpRefList: a list of tempExpRef that actually exist
437  # and weightList: a list of the weight of the associated coadd tempExp
438  # and imageScalerList: a list of scale factors for the associated coadd tempExp
439  tempExpRefList = []
440  weightList = []
441  imageScalerList = []
442  tempExpName = self.getTempExpDatasetName(self.warpType)
443  for tempExpRef in refList:
444  if not tempExpRef.datasetExists(tempExpName):
445  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
446  continue
447 
448  tempExp = tempExpRef.get(tempExpName, immediate=True)
449  maskedImage = tempExp.getMaskedImage()
450  imageScaler = self.scaleZeroPoint.computeImageScaler(
451  exposure=tempExp,
452  dataRef=tempExpRef,
453  )
454  try:
455  imageScaler.scaleMaskedImage(maskedImage)
456  except Exception as e:
457  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
458  continue
459  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
460  afwMath.MEANCLIP, statsCtrl)
461  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
462  weight = 1.0 / float(meanVar)
463  if not numpy.isfinite(weight):
464  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
465  continue
466  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
467 
468  del maskedImage
469  del tempExp
470 
471  tempExpRefList.append(tempExpRef)
472  weightList.append(weight)
473  imageScalerList.append(imageScaler)
474 
475  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
476  imageScalerList=imageScalerList)
477 
478  def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None):
479  """!
480  \brief Perform background matching on the prepared inputs
481 
482  Each Warp has a different background level that must be normalized to a reference level
483  before coaddition. If no reference is provided, the background matcher selects one. If the background
484  matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
485  consistent with the scaled background.
486 
487  \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
488  \param[in] refExpDataRef: Data reference for background reference Warp, or None
489  \param[in] refImageScaler: Image scaler for background reference Warp, or None
490  \return Struct:
491  - tempExprefList: List of data references to warped exposures (coaddTempExps)
492  - weightList: List of weightings
493  - imageScalerList: List of image scalers
494  - backgroundInfoList: result from background matching
495  """
496  try:
497  backgroundInfoList = self.matchBackgrounds.run(
498  expRefList=inputData.tempExpRefList,
499  imageScalerList=inputData.imageScalerList,
500  refExpDataRef=refExpDataRef if not self.config.autoReference else None,
501  refImageScaler=refImageScaler,
502  expDatasetType=self.getTempExpDatasetName(self.warpType),
503  ).backgroundInfoList
504  except Exception as e:
505  self.log.fatal("Cannot match backgrounds: %s", e)
506  raise pipeBase.TaskError("Background matching failed.")
507 
508  newWeightList = []
509  newTempExpRefList = []
510  newBackgroundStructList = []
511  newScaleList = []
512  # the number of good backgrounds may be < than len(tempExpList)
513  # sync these up and correct the weights
514  for tempExpRef, bgInfo, scaler, weight in zip(inputData.tempExpRefList, backgroundInfoList,
515  inputData.imageScalerList, inputData.weightList):
516  if not bgInfo.isReference:
517  # skip exposure if it has no backgroundModel
518  # or if fit was bad
519  if (bgInfo.backgroundModel is None):
520  self.log.info("No background offset model available for %s: skipping", tempExpRef.dataId)
521  continue
522  try:
523  varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
524  except Exception as e:
525  self.log.info("MSE/Var ratio not calculable (%s) for %s: skipping",
526  e, tempExpRef.dataId)
527  continue
528  if not numpy.isfinite(varianceRatio):
529  self.log.info("MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
530  bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
531  continue
532  elif (varianceRatio > self.config.maxMatchResidualRatio):
533  self.log.info("Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
534  varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
535  continue
536  elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
537  self.log.info("Bad fit. RMS %.2f > %.2f for %s: skipping",
538  bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
539  continue
540  newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
541  newTempExpRefList.append(tempExpRef)
542  newBackgroundStructList.append(bgInfo)
543  newScaleList.append(scaler)
544 
545  return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
546  imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
547 
548  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
549  altMaskList=None, doClip=False, mask=None):
550  """!
551  \anchor AssembleCoaddTask.assemble_
552 
553  \brief Assemble a coadd from input warps
554 
555  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
556  large area), the assembly is performed over small areas on the image at a time in order to
557  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
558  \ref assembleSubregion to mean-stack the corresponding subregions from the coaddTempExps (with outlier
559  rejection if config.doSigmaClip=True). Set the edge bits the the coadd mask based on the weight map.
560 
561  \param[in] skyInfo: Patch geometry information, from getSkyInfo
562  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
563  \param[in] imageScalerList: List of image scalers
564  \param[in] weightList: List of weights
565  \param[in] bgInfoList: List of background data from background matching, or None
566  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
567  \param[in] doClip: Use clipping when codding?
568  \param[in] mask: Mask to ignore when coadding
569  \return coadded exposure
570  """
571  tempExpName = self.getTempExpDatasetName(self.warpType)
572  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
573  if mask is None:
574  mask = self.getBadPixelMask()
575 
576  statsCtrl = afwMath.StatisticsControl()
577  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
578  statsCtrl.setNumIter(self.config.clipIter)
579  statsCtrl.setAndMask(mask)
580  statsCtrl.setNanSafe(True)
581  statsCtrl.setWeighted(True)
582  statsCtrl.setCalcErrorFromInputVariance(True)
583  for plane, threshold in self.config.maskPropagationThresholds.items():
584  bit = afwImage.Mask.getMaskPlane(plane)
585  statsCtrl.setMaskPropagationThreshold(bit, threshold)
586 
587  if doClip:
588  statsFlags = afwMath.MEANCLIP
589  else:
590  statsFlags = afwMath.MEAN
591 
592  if bgInfoList is None:
593  bgInfoList = [None]*len(tempExpRefList)
594 
595  if altMaskList is None:
596  altMaskList = [None]*len(tempExpRefList)
597 
598  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
599  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
600  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
601  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
602  coaddMaskedImage = coaddExposure.getMaskedImage()
603  subregionSizeArr = self.config.subregionSize
604  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
605  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
606  try:
607  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
608  weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
609  except Exception as e:
610  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
611 
612  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
613 
614  return coaddExposure
615 
616  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
617  """!
618  \brief Set the metadata for the coadd
619 
620  This basic implementation simply sets the filter from the
621  first input.
622 
623  \param[in] coaddExposure: The target image for the coadd
624  \param[in] tempExpRefList: List of data references to tempExp
625  \param[in] weightList: List of weights
626  """
627  assert len(tempExpRefList) == len(weightList), "Length mismatch"
628  tempExpName = self.getTempExpDatasetName(self.warpType)
629  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
630  # (and we need more than just the PropertySet that contains the header), which is not possible
631  # with the current butler (see #2777).
632  tempExpList = [tempExpRef.get(tempExpName + "_sub",
633  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
634  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
635  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
636 
637  coaddExposure.setFilter(tempExpList[0].getFilter())
638  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
639  coaddInputs.ccds.reserve(numCcds)
640  coaddInputs.visits.reserve(len(tempExpList))
641 
642  for tempExp, weight in zip(tempExpList, weightList):
643  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
644  coaddInputs.visits.sort()
645  if self.warpType == "psfMatched":
646  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
647  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
648  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
649  # having the maximum width (sufficient because square)
650  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
651  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
652  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
653  else:
654  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
655  self.config.coaddPsf.makeControl())
656  coaddExposure.setPsf(psf)
657  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
658  coaddExposure.getWcs())
659  coaddExposure.getInfo().setApCorrMap(apCorrMap)
660 
661  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
662  bgInfoList, altMaskList, statsFlags, statsCtrl):
663  """!
664  \brief Assemble the coadd for a sub-region.
665 
666  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
667  matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
668  planes listed in config.removeMaskPlanes, Finally, mean-stack
669  the actual exposures using \ref afwMath.statisticsStack "statisticsStack" with outlier rejection if
670  config.doSigmaClip=True. Assign the stacked subregion back to the coadd.
671 
672  \param[in] coaddExposure: The target image for the coadd
673  \param[in] bbox: Sub-region to coadd
674  \param[in] tempExpRefList: List of data reference to tempExp
675  \param[in] imageScalerList: List of image scalers
676  \param[in] weightList: List of weights
677  \param[in] bgInfoList: List of background data from background matching
678  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
679  \param[in] statsFlags: Statistic for coadd
680  \param[in] statsCtrl: Statistics control object for coadd
681  """
682  self.log.debug("Computing coadd over %s", bbox)
683  tempExpName = self.getTempExpDatasetName(self.warpType)
684  coaddMaskedImage = coaddExposure.getMaskedImage()
685  maskedImageList = []
686  for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
687  altMaskList):
688  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
689  maskedImage = exposure.getMaskedImage()
690 
691  if altMask:
692  altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
693  maskedImage.getMask().swap(altMaskSub)
694  imageScaler.scaleMaskedImage(maskedImage)
695 
696  if self.config.doMatchBackgrounds and not bgInfo.isReference:
697  backgroundModel = bgInfo.backgroundModel
698  backgroundImage = backgroundModel.getImage() if \
699  self.matchBackgrounds.config.usePolynomial else \
700  backgroundModel.getImageF()
701  backgroundImage.setXY0(coaddMaskedImage.getXY0())
702  maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
703  var = maskedImage.getVariance()
704  var += (bgInfo.fitRMS)**2
705 
706  if self.config.removeMaskPlanes:
707  mask = maskedImage.getMask()
708  for maskPlane in self.config.removeMaskPlanes:
709  try:
710  mask &= ~mask.getPlaneBitMask(maskPlane)
711  except Exception as e:
712  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.message)
713 
714  maskedImageList.append(maskedImage)
715 
716  with self.timer("stack"):
717  coaddSubregion = afwMath.statisticsStack(
718  maskedImageList, statsFlags, statsCtrl, weightList)
719 
720  coaddMaskedImage.assign(coaddSubregion, bbox)
721 
722  def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList):
723  """!
724  \brief Add metadata from the background matching to the coadd
725 
726  \param[in] coaddExposure: Coadd
727  \param[in] tempExpRefList: List of data references for temp exps to go into coadd
728  \param[in] backgroundInfoList: List of background info, results from background matching
729  """
730  self.log.info("Adding exposure information to metadata")
731  metadata = coaddExposure.getMetadata()
732  metadata.addString("CTExp_SDQA1_DESCRIPTION",
733  "Background matching: Ratio of matchedMSE / diffImVar")
734  for ind, (tempExpRef, backgroundInfo) in enumerate(zip(tempExpRefList, backgroundInfoList)):
735  tempExpStr = '&'.join('%s=%s' % (k, v) for k, v in tempExpRef.dataId.items())
736  if backgroundInfo.isReference:
737  metadata.addString("ReferenceExp_ID", tempExpStr)
738  else:
739  metadata.addString("CTExp_ID_%d" % (ind), tempExpStr)
740  metadata.addDouble("CTExp_SDQA1_%d" % (ind),
741  backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
742  metadata.addDouble("CTExp_SDQA2_%d" % (ind),
743  backgroundInfo.fitRMS)
744 
745  def readBrightObjectMasks(self, dataRef):
746  """Returns None on failure"""
747  try:
748  return dataRef.get("brightObjectMask", immediate=True)
749  except Exception as e:
750  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
751  return None
752 
753  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
754  """Set the bright object masks
755 
756  exposure: Exposure under consideration
757  dataId: Data identifier dict for patch
758  brightObjectMasks: afwTable of bright objects to mask
759  """
760  #
761  # Check the metadata specifying the tract/patch/filter
762  #
763  if brightObjectMasks is None:
764  self.log.warn("Unable to apply bright object mask: none supplied")
765  return
766  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
767  md = brightObjectMasks.table.getMetadata()
768  for k in dataId:
769  if not md.exists(k):
770  self.log.warn("Expected to see %s in metadata", k)
771  else:
772  if md.get(k) != dataId[k]:
773  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
774 
775  mask = exposure.getMaskedImage().getMask()
776  wcs = exposure.getWcs()
777  plateScale = wcs.pixelScale().asArcseconds()
778 
779  for rec in brightObjectMasks:
780  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
781  if rec["type"] == "box":
782  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
783  width = rec["width"].asArcseconds()/plateScale # convert to pixels
784  height = rec["height"].asArcseconds()/plateScale # convert to pixels
785 
786  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
787  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
788 
789  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
790  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
791  spans = afwGeom.SpanSet(bbox)
792  elif rec["type"] == "circle":
793  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
794  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
795  else:
796  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
797  continue
798  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
799 
800  @classmethod
801  def _makeArgumentParser(cls):
802  """!
803  \brief Create an argument parser
804  """
805  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
806  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_directWarp",
807  help="data ID, e.g. --id tract=12345 patch=1,2",
808  ContainerClass=AssembleCoaddDataIdContainer)
809  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
810  ContainerClass=SelectDataIdContainer)
811  return parser
812 
813 
814 def _subBBoxIter(bbox, subregionSize):
815  """!
816  \brief Iterate over subregions of a bbox
817 
818  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
819  \param[in] subregionSize: size of sub-bboxes
820 
821  \return subBBox: next sub-bounding box of size subregionSize or smaller;
822  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
823  but it will never be empty
824  """
825  if bbox.isEmpty():
826  raise RuntimeError("bbox %s is empty" % (bbox,))
827  if subregionSize[0] < 1 or subregionSize[1] < 1:
828  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
829 
830  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
831  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
832  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
833  subBBox.clip(bbox)
834  if subBBox.isEmpty():
835  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
836  (bbox, subregionSize, colShift, rowShift))
837  yield subBBox
838 
839 
840 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
841  """!
842  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
843  """
844 
845  def makeDataRefList(self, namespace):
846  """!
847  \brief Make self.refList from self.idList.
848 
849  Interpret the config.doMatchBackgrounds, config.autoReference,
850  and whether a visit/run supplied.
851  If a visit/run is supplied, config.autoReference is automatically set to False.
852  if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
853 
854  """
855  keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName + "Coadd",
856  level=self.level)
857  keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
858  "Coadd_directWarp", level=self.level)
859 
860  if namespace.config.doMatchBackgrounds:
861  if namespace.config.autoReference: # matcher will pick it's own reference image
862  datasetType = namespace.config.coaddName + "Coadd"
863  validKeys = keysCoadd
864  else:
865  datasetType = namespace.config.coaddName + "Coadd_directWarp"
866  validKeys = keysCoaddTempExp
867  else: # bkg subtracted coadd
868  datasetType = namespace.config.coaddName + "Coadd"
869  validKeys = keysCoadd
870 
871  for dataId in self.idList:
872  # tract and patch are required
873  for key in validKeys:
874  if key not in dataId:
875  raise RuntimeError("--id must include " + key)
876 
877  for key in dataId: # check if users supplied visit/run
878  if (key not in keysCoadd) and (key in keysCoaddTempExp): # user supplied a visit/run
879  if namespace.config.autoReference:
880  # user probably meant: autoReference = False
881  namespace.config.autoReference = False
882  datasetType = namespace.config.coaddName + "Coadd_directWarp"
883  print("Switching config.autoReference to False; applies only to background Matching.")
884  break
885 
886  dataRef = namespace.butler.dataRef(
887  datasetType=datasetType,
888  dataId=dataId,
889  )
890  self.refList.append(dataRef)
891 
892 
893 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
894  """!
895  \brief Function to count the number of pixels with a specific mask in a footprint.
896 
897  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
898  have bitmask set but do not have ignoreMask set. Return the count.
899 
900  \param[in] mask: mask to define intersection region by.
901  \parma[in] footprint: footprint to define the intersection region by.
902  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
903  \param[in] ignoreMask: pixels to not consider.
904  \return count of number of pixels in footprint with specified mask.
905  """
906  bbox = footprint.getBBox()
907  bbox.clip(mask.getBBox(afwImage.PARENT))
908  fp = afwImage.Mask(bbox)
909  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
910  footprint.spans.setMask(fp, bitmask)
911  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
912  (subMask.getArray() & ignoreMask) == 0).sum()
913 
914 
916  """!
917 \anchor SafeClipAssembleCoaddConfig
918 
919 \brief Configuration parameters for the SafeClipAssembleCoaddTask
920  """
921  clipDetection = pexConfig.ConfigurableField(
922  target=SourceDetectionTask,
923  doc="Detect sources on difference between unclipped and clipped coadd")
924  minClipFootOverlap = pexConfig.Field(
925  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
926  dtype=float,
927  default=0.6
928  )
929  minClipFootOverlapSingle = pexConfig.Field(
930  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
931  "clipped when only one visit overlaps",
932  dtype=float,
933  default=0.5
934  )
935  minClipFootOverlapDouble = pexConfig.Field(
936  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
937  "clipped when two visits overlap",
938  dtype=float,
939  default=0.45
940  )
941  maxClipFootOverlapDouble = pexConfig.Field(
942  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
943  "considering two visits",
944  dtype=float,
945  default=0.15
946  )
947  minBigOverlap = pexConfig.Field(
948  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
949  "when labeling clipped footprints",
950  dtype=int,
951  default=100
952  )
953 
954  def setDefaults(self):
955  # The numeric values for these configuration parameters were empirically determined, future work
956  # may further refine them.
957  AssembleCoaddConfig.setDefaults(self)
958  self.clipDetection.doTempLocalBackground = False
959  self.clipDetection.reEstimateBackground = False
960  self.clipDetection.returnOriginalFootprints = False
961  self.clipDetection.thresholdPolarity = "both"
962  self.clipDetection.thresholdValue = 2
963  self.clipDetection.nSigmaToGrow = 2
964  self.clipDetection.minPixels = 4
965  self.clipDetection.isotropicGrow = True
966  self.clipDetection.thresholdType = "pixel_stdev"
967  self.sigmaClip = 1.5
968  self.clipIter = 3
969 
970 ## \addtogroup LSST_task_documentation
971 ## \{
972 ## \page SafeClipAssembleCoaddTask
973 ## \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask"
974 ## \copybrief SafeClipAssembleCoaddTask
975 ## \}
976 
977 
979  """!
980  \anchor SafeClipAssembleCoaddTask_
981 
982  \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
983  with potential artifacts.
984 
985  \section pipe_tasks_assembleCoadd_Contents Contents
986  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
987  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
988  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
989  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
990  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
991  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
992 
993  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
994 
995  \copybrief SafeClipAssembleCoaddTask
996 
997  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
998  SafeClipAssembleCoaddTask subtasks that task.
999  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1000  outliers).
1001  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1002  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1003  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
1004  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
1005  that there is an outlier and ii. this outlier appears on only one or two images.
1006  Such regions will not contribute to the final coadd.
1007  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1008  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1009  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1010  surveys.
1011 
1012  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1013  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1014  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1015 
1016  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1017  \copydoc \_\_init\_\_
1018 
1019  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1020  \copydoc run
1021 
1022  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1023  See \ref SafeClipAssembleCoaddConfig
1024 
1025  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1026  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1027  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1028  files.
1029  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1030  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1031  for further information.
1032 
1033  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1034 
1035  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1036  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1037  '--legacyCoadd'.
1038  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1039  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1040  with a list of coaddTempExps to attempt to coadd (specified using
1041  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1042  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1043  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1044  command line argument:
1045  \code
1046  assembleCoadd.py --help
1047  \endcode
1048  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1049  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1050  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1051  packages.
1052  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1053  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1054  <DL>
1055  <DT>processCcd</DT>
1056  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1057  <DT>makeSkyMap</DT>
1058  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1059  <DT>makeCoaddTempExp</DT>
1060  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1061  </DL>
1062  We can perform all of these steps by running
1063  \code
1064  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1065  \endcode
1066  This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1067  as follows:
1068  \code
1069  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1070  \endcode
1071  This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1072  You may also choose to run:
1073  \code
1074  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1075  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1076  \endcode
1077  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1078  discussed in \ref pipeTasks_multiBand.
1079  """
1080  ConfigClass = SafeClipAssembleCoaddConfig
1081  _DefaultName = "safeClipAssembleCoadd"
1082 
1083  def __init__(self, *args, **kwargs):
1084  """!
1085  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1086  """
1087  AssembleCoaddTask.__init__(self, *args, **kwargs)
1088  schema = afwTable.SourceTable.makeMinimalSchema()
1089  self.makeSubtask("clipDetection", schema=schema)
1090 
1091  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1092  """!
1093  \brief Assemble the coadd for a region
1094 
1095  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1096  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1097  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1098  overlap between the clipped region and a source.
1099  This leaves us with a set of footprints from the difference image that have been identified as having
1100  occured on just one or two individual visits. However, these footprints were generated from a
1101  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1102  footprints acrosss the coadd difference in this process.
1103  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1104  these are big footprints.
1105  Combine the small and big clipped footprints and mark them on a new bad mask plane
1106  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1107  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1108  bad mask plane.
1109 
1110  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1111  parent task.
1112 
1113  @param skyInfo: Patch geometry information, from getSkyInfo
1114  @param tempExpRefList: List of data reference to tempExp
1115  @param imageScalerList: List of image scalers
1116  @param weightList: List of weights
1117  @param bgModelList: List of background models from background matching
1118  return coadd exposure
1119  """
1120  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1121  mask = exp.getMaskedImage().getMask()
1122  mask.addMaskPlane("CLIPPED")
1123 
1124  result = self.detectClip(exp, tempExpRefList)
1125 
1126  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1127 
1128  # Go to individual visits for big footprints
1129  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1130  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1131  bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1132  maskClipValue, maskDetValue)
1133 
1134  # Create mask of the current clipped footprints
1135  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1136  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1137 
1138  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1139  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1140  maskClip |= maskClipBig
1141 
1142  # Assemble coadd from base class, but ignoring CLIPPED pixels (doClip is false)
1143  badMaskPlanes = self.config.badMaskPlanes[:]
1144  badMaskPlanes.append("CLIPPED")
1145  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1146  coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1147  bgModelList, result.tempExpClipList,
1148  doClip=False,
1149  mask=badPixelMask)
1150 
1151  # Set the coadd CLIPPED mask from the footprints since currently pixels that are masked
1152  # do not get propagated
1153  maskExp = coaddExp.getMaskedImage().getMask()
1154  maskExp |= maskClip
1155 
1156  return coaddExp
1157 
1158  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList):
1159  """!
1160  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1161 
1162  Generate a difference image between clipped and unclipped coadds.
1163  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1164  Return the difference image.
1165 
1166  @param skyInfo: Patch geometry information, from getSkyInfo
1167  @param tempExpRefList: List of data reference to tempExp
1168  @param imageScalerList: List of image scalers
1169  @param weightList: List of weights
1170  @param bgModelList: List of background models from background matching
1171  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1172  """
1173  # Build the unclipped coadd
1174  coaddMean = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1175  bgModelList, doClip=False)
1176 
1177  # Build the clipped coadd
1178  coaddClip = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1179  bgModelList, doClip=True)
1180 
1181  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1182  coaddDiff -= coaddClip.getMaskedImage()
1183  exp = afwImage.ExposureF(coaddDiff)
1184  exp.setPsf(coaddMean.getPsf())
1185  return exp
1186 
1187  def detectClip(self, exp, tempExpRefList):
1188  """!
1189  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1190 
1191  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1192  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1193  fraction to thresholds set in the config.
1194  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1195  two).
1196  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1197  the coaddTempExp.
1198  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1199  with the clipped footprints and a list of new masks for the coaddTempExps.
1200 
1201  \param[in] exp: Exposure to run detection on
1202  \param[in] tempExpRefList: List of data reference to tempExp
1203  \return struct containing:
1204  - clippedFootprints: list of clipped footprints
1205  - clippedIndices: indices for each clippedFootprint in tempExpRefList
1206  - tempExpClipList: list of new masks for tempExp
1207  """
1208  mask = exp.getMaskedImage().getMask()
1209  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1210  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1211  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1212  # Merge positive and negative together footprints together
1213  fpSet.positive.merge(fpSet.negative)
1214  footprints = fpSet.positive
1215  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1216  ignoreMask = self.getBadPixelMask()
1217 
1218  clipFootprints = []
1219  clipIndices = []
1220 
1221  # build a list with a mask for each visit which can be modified with clipping information
1222  tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.warpType),
1223  immediate=True).getMaskedImage().getMask() for
1224  tmpExpRef in tempExpRefList]
1225 
1226  for footprint in footprints.getFootprints():
1227  nPixel = footprint.getArea()
1228  overlap = [] # hold the overlap with each visit
1229  maskList = [] # which visit mask match
1230  indexList = [] # index of visit in global list
1231  for i, tmpExpMask in enumerate(tempExpClipList):
1232  # Determine the overlap with the footprint
1233  ignore = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1234  overlapDet = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1235  totPixel = nPixel - ignore
1236 
1237  # If we have more bad pixels than detection skip
1238  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1239  continue
1240  overlap.append(overlapDet/float(totPixel))
1241  maskList.append(tmpExpMask)
1242  indexList.append(i)
1243 
1244  overlap = numpy.array(overlap)
1245  if not len(overlap):
1246  continue
1247 
1248  keep = False # Should this footprint be marked as clipped?
1249  keepIndex = [] # Which tempExps does the clipped footprint belong to
1250 
1251  # If footprint only has one overlap use a lower threshold
1252  if len(overlap) == 1:
1253  if overlap[0] > self.config.minClipFootOverlapSingle:
1254  keep = True
1255  keepIndex = [0]
1256  else:
1257  # This is the general case where only visit should be clipped
1258  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1259  if len(clipIndex) == 1:
1260  keep = True
1261  keepIndex = [clipIndex[0]]
1262 
1263  # Test if there are clipped objects that overlap two different visits
1264  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1265  if len(clipIndex) == 2 and len(overlap) > 3:
1266  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1267  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1268  keep = True
1269  keepIndex = clipIndex
1270 
1271  if not keep:
1272  continue
1273 
1274  for index in keepIndex:
1275  footprint.spans.setMask(maskList[index], maskClipValue)
1276 
1277  clipIndices.append(numpy.array(indexList)[keepIndex])
1278  clipFootprints.append(footprint)
1279 
1280  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1281  tempExpClipList=tempExpClipList)
1282 
1283  def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1284  """!
1285  \brief Find footprints from individual tempExp footprints for large footprints.
1286 
1287  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1288  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1289  significantly with each source in all the coaddTempExps.
1290 
1291  \param[in] tempExpClipList: List of tempExp masks with clipping information
1292  \param[in] clipFootprints: List of clipped footprints
1293  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1294  \param[in] maskClipValue: Mask value of clipped pixels
1295  \param[in] maskClipValue: Mask value of detected pixels
1296  \return list of big footprints
1297  """
1298  bigFootprintsCoadd = []
1299  ignoreMask = self.getBadPixelMask()
1300  for index, tmpExpMask in enumerate(tempExpClipList):
1301 
1302  # Create list of footprints from the DETECTED pixels
1303  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1304  afwImage.PARENT, True)
1305  maskVisitDet &= maskDetValue
1306  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1307 
1308  # build a mask of clipped footprints that are in this visit
1309  clippedFootprintsVisit = []
1310  for foot, clipIndex in zip(clipFootprints, clipIndices):
1311  if index not in clipIndex:
1312  continue
1313  clippedFootprintsVisit.append(foot)
1314  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1315  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1316 
1317  bigFootprintsVisit = []
1318  for foot in visitFootprints.getFootprints():
1319  if foot.getArea() < self.config.minBigOverlap:
1320  continue
1321  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1322  if nCount > self.config.minBigOverlap:
1323  bigFootprintsVisit.append(foot)
1324  bigFootprintsCoadd.append(foot)
1325 
1326  # Update single visit masks
1327  maskVisitClip.clearAllMaskPlanes()
1328  afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1329  tmpExpMask |= maskVisitClip
1330 
1331  return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of Warps.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip &amp; flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...