lsst.pipe.tasks  13.0-54-gc325bc5
 All Classes Namespaces Files Functions Variables Groups Pages
assembleCoadd.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import zip
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 import numpy
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.afw.detection as afwDetect
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 from .coaddBase import CoaddBaseTask, SelectDataIdContainer
39 from .interpImage import InterpImageTask
40 from .matchBackgrounds import MatchBackgroundsTask
41 from .scaleZeroPoint import ScaleZeroPointTask
42 from .coaddHelpers import groupPatchExposures, getGroupDataRef
43 from lsst.meas.algorithms import SourceDetectionTask
44 
45 __all__ = ["AssembleCoaddTask", "SafeClipAssembleCoaddTask"]
46 
47 
48 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass):
49  """!
50 \anchor AssembleCoaddConfig_
51 
52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53  """
54  subregionSize = pexConfig.ListField(
55  dtype=int,
56  doc="Width, height of stack subregion size; "
57  "make small enough that a full stack of images will fit into memory at once.",
58  length=2,
59  default=(2000, 2000),
60  )
61  statistic = pexConfig.Field(
62  dtype=str,
63  doc="Main stacking statistic for aggregating over the epochs.",
64  default="MEANCLIP",
65  )
66  doSigmaClip = pexConfig.Field(
67  dtype=bool,
68  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
69  default=False,
70  )
71  sigmaClip = pexConfig.Field(
72  dtype=float,
73  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
74  default=3.0,
75  )
76  clipIter = pexConfig.Field(
77  dtype=int,
78  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
79  default=2,
80  )
81  scaleZeroPoint = pexConfig.ConfigurableField(
82  target=ScaleZeroPointTask,
83  doc="Task to adjust the photometric zero point of the coadd temp exposures",
84  )
85  doInterp = pexConfig.Field(
86  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
87  dtype=bool,
88  default=True,
89  )
90  interpImage = pexConfig.ConfigurableField(
91  target=InterpImageTask,
92  doc="Task to interpolate (and extrapolate) over NaN pixels",
93  )
94  matchBackgrounds = pexConfig.ConfigurableField(
95  target=MatchBackgroundsTask,
96  doc="Task to match backgrounds",
97  )
98  maxMatchResidualRatio = pexConfig.Field(
99  doc="Maximum ratio of the mean squared error of the background matching model to the variance "
100  "of the difference in backgrounds",
101  dtype=float,
102  default=1.1
103  )
104  maxMatchResidualRMS = pexConfig.Field(
105  doc="Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
106  dtype=float,
107  default=1.0
108  )
109  doWrite = pexConfig.Field(
110  doc="Persist coadd?",
111  dtype=bool,
112  default=True,
113  )
114  doMatchBackgrounds = pexConfig.Field(
115  doc="Match backgrounds of coadd temp exposures before coadding them? "
116  "If False, the coadd temp expsosures must already have been background subtracted or matched",
117  dtype=bool,
118  default=True,
119  )
120  autoReference = pexConfig.Field(
121  doc="Automatically select the coadd temp exposure to use as a reference for background matching? "
122  "Ignored if doMatchBackgrounds false. "
123  "If False you must specify the reference temp exposure as the data Id",
124  dtype=bool,
125  default=True,
126  )
127  maskPropagationThresholds = pexConfig.DictField(
128  keytype=str,
129  itemtype=float,
130  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
131  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
132  "would have contributed exceeds this value."),
133  default={"SAT": 0.1},
134  )
135  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["CROSSTALK", "NOT_DEBLENDED"],
136  doc="Mask planes to remove before coadding")
137  #
138  # N.b. These configuration options only set the bitplane config.brightObjectMaskName
139  # To make this useful you *must* also configure the flags.pixel algorithm, for example
140  # by adding
141  # config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
142  # config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
143  # to your measureCoaddSources.py and forcedPhotCoadd.py config overrides
144  #
145  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
146  doc="Set mask and flag bits for bright objects?")
147  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
148  doc="Name of mask bit used for bright objects")
149 
150  coaddPsf = pexConfig.ConfigField(
151  doc="Configuration for CoaddPsf",
152  dtype=measAlg.CoaddPsfConfig,
153  )
154 
155  def setDefaults(self):
156  CoaddBaseTask.ConfigClass.setDefaults(self)
157  self.badMaskPlanes = ["NO_DATA", "BAD", "CR", ]
158 
159  def validate(self):
160  CoaddBaseTask.ConfigClass.validate(self)
161  if self.makeDirect and self.makePsfMatched:
162  raise ValueError("Currently, assembleCoadd can only make either Direct or PsfMatched Coadds "
163  "at a time. Set either makeDirect or makePsfMatched to False")
164  if self.doSigmaClip and self.statistic != "MEANCLIP":
165  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
166  self.statistic = "MEANCLIP"
167  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
168  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
169  "compute and set a non-zero coadd variance estimate." % (self.statistic))
170 
171  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
172  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
173  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
174  if str(k) not in unstackableStats]
175  raise ValueError("statistic %s is not allowed. Please choose one of %s."
176  % (self.statistic, stackableStats))
177 # \addtogroup LSST_task_documentation
178 # \{
179 # \page AssembleCoaddTask
180 # \ref AssembleCoaddTask_ "AssembleCoaddTask"
181 # \copybrief AssembleCoaddTask
182 # \}
183 
184 class AssembleCoaddTask(CoaddBaseTask):
185  """!
186 \anchor AssembleCoaddTask_
187 
188 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
189 
190 \section pipe_tasks_assembleCoadd_Contents Contents
191  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
192  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
193  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
194  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
195  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
196  - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
197 
198 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
199 
200 \copybrief AssembleCoaddTask_
201 
202 We want to assemble a coadded image from a set of Warps (also called
203 coadded temporary exposures or coaddTempExps.
204 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
205 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
206 Warps that cover the specified patch (pointed at by dataRef).
207 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
208 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
209 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
210 When background matching is enabled, the task may be configured to automatically select a reference exposure
211 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
212 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
213 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
214 config.makePsfMatched set which of the warp types will be coadded.
215 The coadd is computed as a mean with optional outlier rejection.
216 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
217 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
218 
219 AssembleCoaddTask uses several sub-tasks. These are
220 <DL>
221  <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
222  <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
223  <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
224  <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
225  not provided).</DD>
226  <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
227  <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
228 </DL>
229 You can retarget these subtasks if you wish.
230 
231 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
232 \copydoc \_\_init\_\_
233 
234 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
235 \copydoc run
236 
237 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
238 See \ref AssembleCoaddConfig_
239 
240 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
241 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
242 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
243 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
244 the documetation for the subtasks for further information.
245 
246 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
247 
248 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
249 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
250 a data reference to the tract patch and filter to be coadded (specified using
251 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
252 Warps to attempt to coadd (specified using
253 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
254 that cover the specified tract and patch will be coadded. A list of the available optional
255 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
256 \code
257 assembleCoadd.py --help
258 \endcode
259 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
260 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
261 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
262 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
263 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
264 <DL>
265  <DT>processCcd</DT>
266  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
267  <DT>makeSkyMap</DT>
268  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
269  <DT>makeCoaddTempExp</DT>
270  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
271 </DL>
272 We can perform all of these steps by running
273 \code
274 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
275 \endcode
276 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
277 follows:
278 \code
279 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
280 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
281 You may also choose to run:
282 \code
283 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
284 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
285 \endcode
286 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
287 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
288 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
289  """
290  ConfigClass = AssembleCoaddConfig
291  _DefaultName = "assembleCoadd"
292 
293  def __init__(self, *args, **kwargs):
294  """!
295  \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
296  \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
297  """
298  CoaddBaseTask.__init__(self, *args, **kwargs)
299  self.makeSubtask("interpImage")
300  self.makeSubtask("matchBackgrounds")
301  self.makeSubtask("scaleZeroPoint")
302 
303  if self.config.doMaskBrightObjects:
304  mask = afwImage.Mask()
305  try:
306  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
307  except pexExceptions.LsstCppException:
308  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
309  mask.getMaskPlaneDict().keys())
310  del mask
311 
312  if self.config.makeDirect:
313  self.warpType = "direct"
314  elif self.config.makePsfMatched:
315  self.warpType = "psfMatched"
316  else:
317  raise ValueError("Neither makeDirect nor makePsfMatched configs are True")
318 
319  @pipeBase.timeMethod
320  def run(self, dataRef, selectDataList=[]):
321  """!
322  \brief Assemble a coadd from a set of Warps
323 
324  Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
325  match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
326  Warps if the background has not already been removed. Assemble the Warps using
327  \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
328  exposure.
329 
330  \anchor runParams
331  \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
332  (if config.autoReference=False). Used to access the following data products:
333  - [in] self.config.coaddName + "Coadd_skyMap"
334  - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
335  - [out] self.config.coaddName + "Coadd"
336  \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
337  selected from this list based on overlap with the patch defined by dataRef.
338 
339  \return a pipeBase.Struct with fields:
340  - coaddExposure: coadded exposure
341  """
342  skyInfo = self.getSkyInfo(dataRef)
343  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
344  if len(calExpRefList) == 0:
345  self.log.warn("No exposures to coadd")
346  return
347  self.log.info("Coadding %d exposures", len(calExpRefList))
348 
349  tempExpRefList = self.getTempExpRefList(dataRef, calExpRefList)
350  inputData = self.prepareInputs(tempExpRefList)
351  self.log.info("Found %d %s", len(inputData.tempExpRefList),
352  self.getTempExpDatasetName(self.warpType))
353  if len(inputData.tempExpRefList) == 0:
354  self.log.warn("No coadd temporary exposures found")
355  return
356  if self.config.doMatchBackgrounds:
357  refImageScaler = self.getBackgroundReferenceScaler(dataRef)
358  inputData = self.backgroundMatching(inputData, dataRef, refImageScaler)
359  if len(inputData.tempExpRefList) == 0:
360  self.log.warn("No valid background models")
361  return
362 
363  coaddExp = self.assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
364  inputData.weightList,
365  inputData.backgroundInfoList if self.config.doMatchBackgrounds else None)
366  if self.config.doMatchBackgrounds:
367  self.addBackgroundMatchingMetadata(coaddExp, inputData.tempExpRefList,
368  inputData.backgroundInfoList)
369 
370  if self.config.doInterp:
371  self.interpImage.run(coaddExp.getMaskedImage(), planeName="NO_DATA")
372  # The variance must be positive; work around for DM-3201.
373  varArray = coaddExp.getMaskedImage().getVariance().getArray()
374  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
375 
376  if self.config.doMaskBrightObjects:
377  brightObjectMasks = self.readBrightObjectMasks(dataRef)
378  self.setBrightObjectMasks(coaddExp, dataRef.dataId, brightObjectMasks)
379 
380  if self.config.doWrite:
381  self.log.info("Persisting %s" % self.getCoaddDatasetName(self.warpType))
382  dataRef.put(coaddExp, self.getCoaddDatasetName(self.warpType))
383 
384  return pipeBase.Struct(coaddExposure=coaddExp)
385 
386  def getTempExpRefList(self, patchRef, calExpRefList):
387  """!
388  \brief Generate list data references corresponding to warped exposures that lie within the
389  patch to be coadded.
390 
391  \param[in] patchRef: Data reference for patch
392  \param[in] calExpRefList: List of data references for input calexps
393  \return List of Warp/CoaddTempExp data references
394  """
395  butler = patchRef.getButler()
396  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
397  self.getTempExpDatasetName(self.warpType))
398  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
399  g, groupData.keys) for
400  g in groupData.groups.keys()]
401  return tempExpRefList
402 
403  def getBackgroundReferenceScaler(self, dataRef):
404  """!
405  \brief Construct an image scaler for the background reference frame
406 
407  Each Warp has a different background level. A reference background level must be chosen before
408  coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
409  this routine is a no-op and None is returned. Otherwise, use the
410  \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
411  reference image and return it.
412 
413  \param[in] dataRef: Data reference for the background reference frame, or None
414  \return image scaler, or None
415  """
416  if self.config.autoReference:
417  return None
418 
419  # We've been given the data reference
420  dataset = self.getTempExpDatasetName(self.warpType)
421  if not dataRef.datasetExists(dataset):
422  raise RuntimeError("Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
423 
424  refExposure = dataRef.get(self.getTempExpDatasetName(self.warpType), immediate=True)
425  refImageScaler = self.scaleZeroPoint.computeImageScaler(
426  exposure=refExposure,
427  dataRef=dataRef,
428  )
429  return refImageScaler
430 
431  def prepareInputs(self, refList):
432  """!
433  \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
434  for the photometric zero point.
435 
436  Each Warp has its own photometric zeropoint and background variance. Before coadding these
437  Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
438  weight for each Warp.
439 
440  \param[in] refList: List of data references to tempExp
441  \return Struct:
442  - tempExprefList: List of data references to tempExp
443  - weightList: List of weightings
444  - imageScalerList: List of image scalers
445  """
446  statsCtrl = afwMath.StatisticsControl()
447  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
448  statsCtrl.setNumIter(self.config.clipIter)
449  statsCtrl.setAndMask(self.getBadPixelMask())
450  statsCtrl.setNanSafe(True)
451 
452  # compute tempExpRefList: a list of tempExpRef that actually exist
453  # and weightList: a list of the weight of the associated coadd tempExp
454  # and imageScalerList: a list of scale factors for the associated coadd tempExp
455  tempExpRefList = []
456  weightList = []
457  imageScalerList = []
458  tempExpName = self.getTempExpDatasetName(self.warpType)
459  for tempExpRef in refList:
460  if not tempExpRef.datasetExists(tempExpName):
461  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
462  continue
463 
464  tempExp = tempExpRef.get(tempExpName, immediate=True)
465  maskedImage = tempExp.getMaskedImage()
466  imageScaler = self.scaleZeroPoint.computeImageScaler(
467  exposure=tempExp,
468  dataRef=tempExpRef,
469  )
470  try:
471  imageScaler.scaleMaskedImage(maskedImage)
472  except Exception as e:
473  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
474  continue
475  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
476  afwMath.MEANCLIP, statsCtrl)
477  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
478  weight = 1.0 / float(meanVar)
479  if not numpy.isfinite(weight):
480  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
481  continue
482  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
483 
484  del maskedImage
485  del tempExp
486 
487  tempExpRefList.append(tempExpRef)
488  weightList.append(weight)
489  imageScalerList.append(imageScaler)
490 
491  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
492  imageScalerList=imageScalerList)
493 
494  def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None):
495  """!
496  \brief Perform background matching on the prepared inputs
497 
498  Each Warp has a different background level that must be normalized to a reference level
499  before coaddition. If no reference is provided, the background matcher selects one. If the background
500  matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
501  consistent with the scaled background.
502 
503  \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
504  \param[in] refExpDataRef: Data reference for background reference Warp, or None
505  \param[in] refImageScaler: Image scaler for background reference Warp, or None
506  \return Struct:
507  - tempExprefList: List of data references to warped exposures (coaddTempExps)
508  - weightList: List of weightings
509  - imageScalerList: List of image scalers
510  - backgroundInfoList: result from background matching
511  """
512  try:
513  backgroundInfoList = self.matchBackgrounds.run(
514  expRefList=inputData.tempExpRefList,
515  imageScalerList=inputData.imageScalerList,
516  refExpDataRef=refExpDataRef if not self.config.autoReference else None,
517  refImageScaler=refImageScaler,
518  expDatasetType=self.getTempExpDatasetName(self.warpType),
519  ).backgroundInfoList
520  except Exception as e:
521  self.log.fatal("Cannot match backgrounds: %s", e)
522  raise pipeBase.TaskError("Background matching failed.")
523 
524  newWeightList = []
525  newTempExpRefList = []
526  newBackgroundStructList = []
527  newScaleList = []
528  # the number of good backgrounds may be < than len(tempExpList)
529  # sync these up and correct the weights
530  for tempExpRef, bgInfo, scaler, weight in zip(inputData.tempExpRefList, backgroundInfoList,
531  inputData.imageScalerList, inputData.weightList):
532  if not bgInfo.isReference:
533  # skip exposure if it has no backgroundModel
534  # or if fit was bad
535  if (bgInfo.backgroundModel is None):
536  self.log.info("No background offset model available for %s: skipping", tempExpRef.dataId)
537  continue
538  try:
539  varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
540  except Exception as e:
541  self.log.info("MSE/Var ratio not calculable (%s) for %s: skipping",
542  e, tempExpRef.dataId)
543  continue
544  if not numpy.isfinite(varianceRatio):
545  self.log.info("MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
546  bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
547  continue
548  elif (varianceRatio > self.config.maxMatchResidualRatio):
549  self.log.info("Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
550  varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
551  continue
552  elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
553  self.log.info("Bad fit. RMS %.2f > %.2f for %s: skipping",
554  bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
555  continue
556  newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
557  newTempExpRefList.append(tempExpRef)
558  newBackgroundStructList.append(bgInfo)
559  newScaleList.append(scaler)
560 
561  return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
562  imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
563 
564  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
565  altMaskList=None, mask=None):
566  """!
567  \anchor AssembleCoaddTask.assemble_
568 
569  \brief Assemble a coadd from input warps
570 
571  Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
572  large area), the assembly is performed over small areas on the image at a time in order to
573  conserve memory usage. Iterate over subregions within the outer bbox of the patch using
574  \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
575  statistic specified. Set the edge bits the coadd mask based on the weight map.
576 
577  \param[in] skyInfo: Patch geometry information, from getSkyInfo
578  \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
579  \param[in] imageScalerList: List of image scalers
580  \param[in] weightList: List of weights
581  \param[in] bgInfoList: List of background data from background matching, or None
582  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
583  \param[in] mask: Mask to ignore when coadding
584  \return coadded exposure
585  """
586  tempExpName = self.getTempExpDatasetName(self.warpType)
587  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
588  if mask is None:
589  mask = self.getBadPixelMask()
590 
591  statsCtrl = afwMath.StatisticsControl()
592  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
593  statsCtrl.setNumIter(self.config.clipIter)
594  statsCtrl.setAndMask(mask)
595  statsCtrl.setNanSafe(True)
596  statsCtrl.setWeighted(True)
597  statsCtrl.setCalcErrorFromInputVariance(True)
598  for plane, threshold in self.config.maskPropagationThresholds.items():
599  bit = afwImage.Mask.getMaskPlane(plane)
600  statsCtrl.setMaskPropagationThreshold(bit, threshold)
601 
602  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
603 
604  if bgInfoList is None:
605  bgInfoList = [None]*len(tempExpRefList)
606 
607  if altMaskList is None:
608  altMaskList = [None]*len(tempExpRefList)
609 
610  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
611  coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
612  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
613  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
614  coaddMaskedImage = coaddExposure.getMaskedImage()
615  subregionSizeArr = self.config.subregionSize
616  subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
617  for subBBox in _subBBoxIter(skyInfo.bbox, subregionSize):
618  try:
619  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
620  weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
621  except Exception as e:
622  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
623 
624  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
625 
626  return coaddExposure
627 
628  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
629  """!
630  \brief Set the metadata for the coadd
631 
632  This basic implementation simply sets the filter from the
633  first input.
634 
635  \param[in] coaddExposure: The target image for the coadd
636  \param[in] tempExpRefList: List of data references to tempExp
637  \param[in] weightList: List of weights
638  """
639  assert len(tempExpRefList) == len(weightList), "Length mismatch"
640  tempExpName = self.getTempExpDatasetName(self.warpType)
641  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
642  # (and we need more than just the PropertySet that contains the header), which is not possible
643  # with the current butler (see #2777).
644  tempExpList = [tempExpRef.get(tempExpName + "_sub",
645  bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
646  imageOrigin="LOCAL", immediate=True) for tempExpRef in tempExpRefList]
647  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
648 
649  coaddExposure.setFilter(tempExpList[0].getFilter())
650  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
651  coaddInputs.ccds.reserve(numCcds)
652  coaddInputs.visits.reserve(len(tempExpList))
653 
654  for tempExp, weight in zip(tempExpList, weightList):
655  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
656  coaddInputs.visits.sort()
657  if self.warpType == "psfMatched":
658  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
659  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
660  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
661  # having the maximum width (sufficient because square)
662  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
663  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
664  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
665  else:
666  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
667  self.config.coaddPsf.makeControl())
668  coaddExposure.setPsf(psf)
669  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
670  coaddExposure.getWcs())
671  coaddExposure.getInfo().setApCorrMap(apCorrMap)
672 
673  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
674  bgInfoList, altMaskList, statsFlags, statsCtrl):
675  """!
676  \brief Assemble the coadd for a sub-region.
677 
678  For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
679  matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
680  planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
681  \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
682  by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
683  afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
684  are specified by statsCtrl. Assign the stacked subregion back to the coadd.
685 
686  \param[in] coaddExposure: The target image for the coadd
687  \param[in] bbox: Sub-region to coadd
688  \param[in] tempExpRefList: List of data reference to tempExp
689  \param[in] imageScalerList: List of image scalers
690  \param[in] weightList: List of weights
691  \param[in] bgInfoList: List of background data from background matching
692  \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
693  \param[in] statsFlags: afwMath.Property object for statistic for coadd
694  \param[in] statsCtrl: Statistics control object for coadd
695  """
696  self.log.debug("Computing coadd over %s", bbox)
697  tempExpName = self.getTempExpDatasetName(self.warpType)
698  coaddMaskedImage = coaddExposure.getMaskedImage()
699  maskedImageList = []
700  for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
701  altMaskList):
702  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
703  maskedImage = exposure.getMaskedImage()
704 
705  if altMask:
706  altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
707  maskedImage.getMask().swap(altMaskSub)
708  imageScaler.scaleMaskedImage(maskedImage)
709 
710  if self.config.doMatchBackgrounds and not bgInfo.isReference:
711  backgroundModel = bgInfo.backgroundModel
712  backgroundImage = backgroundModel.getImage() if \
713  self.matchBackgrounds.config.usePolynomial else \
714  backgroundModel.getImageF()
715  backgroundImage.setXY0(coaddMaskedImage.getXY0())
716  maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
717  var = maskedImage.getVariance()
718  var += (bgInfo.fitRMS)**2
719 
720  if self.config.removeMaskPlanes:
721  mask = maskedImage.getMask()
722  for maskPlane in self.config.removeMaskPlanes:
723  try:
724  mask &= ~mask.getPlaneBitMask(maskPlane)
725  except Exception as e:
726  self.log.warn("Unable to remove mask plane %s: %s", maskPlane, e.message)
727 
728  maskedImageList.append(maskedImage)
729 
730  with self.timer("stack"):
731  coaddSubregion = afwMath.statisticsStack(
732  maskedImageList, statsFlags, statsCtrl, weightList)
733 
734  coaddMaskedImage.assign(coaddSubregion, bbox)
735 
736  def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList):
737  """!
738  \brief Add metadata from the background matching to the coadd
739 
740  \param[in] coaddExposure: Coadd
741  \param[in] tempExpRefList: List of data references for temp exps to go into coadd
742  \param[in] backgroundInfoList: List of background info, results from background matching
743  """
744  self.log.info("Adding exposure information to metadata")
745  metadata = coaddExposure.getMetadata()
746  metadata.addString("CTExp_SDQA1_DESCRIPTION",
747  "Background matching: Ratio of matchedMSE / diffImVar")
748  for ind, (tempExpRef, backgroundInfo) in enumerate(zip(tempExpRefList, backgroundInfoList)):
749  tempExpStr = '&'.join('%s=%s' % (k, v) for k, v in tempExpRef.dataId.items())
750  if backgroundInfo.isReference:
751  metadata.addString("ReferenceExp_ID", tempExpStr)
752  else:
753  metadata.addString("CTExp_ID_%d" % (ind), tempExpStr)
754  metadata.addDouble("CTExp_SDQA1_%d" % (ind),
755  backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
756  metadata.addDouble("CTExp_SDQA2_%d" % (ind),
757  backgroundInfo.fitRMS)
758 
759  def readBrightObjectMasks(self, dataRef):
760  """Returns None on failure"""
761  try:
762  return dataRef.get("brightObjectMask", immediate=True)
763  except Exception as e:
764  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
765  return None
766 
767  def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks):
768  """Set the bright object masks
769 
770  exposure: Exposure under consideration
771  dataId: Data identifier dict for patch
772  brightObjectMasks: afwTable of bright objects to mask
773  """
774  #
775  # Check the metadata specifying the tract/patch/filter
776  #
777  if brightObjectMasks is None:
778  self.log.warn("Unable to apply bright object mask: none supplied")
779  return
780  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
781  md = brightObjectMasks.table.getMetadata()
782  for k in dataId:
783  if not md.exists(k):
784  self.log.warn("Expected to see %s in metadata", k)
785  else:
786  if md.get(k) != dataId[k]:
787  self.log.warn("Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
788 
789  mask = exposure.getMaskedImage().getMask()
790  wcs = exposure.getWcs()
791  plateScale = wcs.pixelScale().asArcseconds()
792 
793  for rec in brightObjectMasks:
794  center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
795  if rec["type"] == "box":
796  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
797  width = rec["width"].asArcseconds()/plateScale # convert to pixels
798  height = rec["height"].asArcseconds()/plateScale # convert to pixels
799 
800  halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
801  bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
802 
803  bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
804  afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
805  spans = afwGeom.SpanSet(bbox)
806  elif rec["type"] == "circle":
807  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
808  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
809  else:
810  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
811  continue
812  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
813 
814  @classmethod
815  def _makeArgumentParser(cls):
816  """!
817  \brief Create an argument parser
818  """
819  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
820  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_directWarp",
821  help="data ID, e.g. --id tract=12345 patch=1,2",
822  ContainerClass=AssembleCoaddDataIdContainer)
823  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
824  ContainerClass=SelectDataIdContainer)
825  return parser
826 
827 
828 def _subBBoxIter(bbox, subregionSize):
829  """!
830  \brief Iterate over subregions of a bbox
831 
832  \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
833  \param[in] subregionSize: size of sub-bboxes
834 
835  \return subBBox: next sub-bounding box of size subregionSize or smaller;
836  each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
837  but it will never be empty
838  """
839  if bbox.isEmpty():
840  raise RuntimeError("bbox %s is empty" % (bbox,))
841  if subregionSize[0] < 1 or subregionSize[1] < 1:
842  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
843 
844  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
845  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
846  subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
847  subBBox.clip(bbox)
848  if subBBox.isEmpty():
849  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
850  (bbox, subregionSize, colShift, rowShift))
851  yield subBBox
852 
853 
854 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
855  """!
856  \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
857  """
858 
859  def makeDataRefList(self, namespace):
860  """!
861  \brief Make self.refList from self.idList.
862 
863  Interpret the config.doMatchBackgrounds, config.autoReference,
864  and whether a visit/run supplied.
865  If a visit/run is supplied, config.autoReference is automatically set to False.
866  if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
867 
868  """
869  keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName + "Coadd",
870  level=self.level)
871  keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
872  "Coadd_directWarp", level=self.level)
873 
874  if namespace.config.doMatchBackgrounds:
875  if namespace.config.autoReference: # matcher will pick it's own reference image
876  datasetType = namespace.config.coaddName + "Coadd"
877  validKeys = keysCoadd
878  else:
879  datasetType = namespace.config.coaddName + "Coadd_directWarp"
880  validKeys = keysCoaddTempExp
881  else: # bkg subtracted coadd
882  datasetType = namespace.config.coaddName + "Coadd"
883  validKeys = keysCoadd
884 
885  for dataId in self.idList:
886  # tract and patch are required
887  for key in validKeys:
888  if key not in dataId:
889  raise RuntimeError("--id must include " + key)
890 
891  for key in dataId: # check if users supplied visit/run
892  if (key not in keysCoadd) and (key in keysCoaddTempExp): # user supplied a visit/run
893  if namespace.config.autoReference:
894  # user probably meant: autoReference = False
895  namespace.config.autoReference = False
896  datasetType = namespace.config.coaddName + "Coadd_directWarp"
897  print("Switching config.autoReference to False; applies only to background Matching.")
898  break
899 
900  dataRef = namespace.butler.dataRef(
901  datasetType=datasetType,
902  dataId=dataId,
903  )
904  self.refList.append(dataRef)
905 
906 
907 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
908  """!
909  \brief Function to count the number of pixels with a specific mask in a footprint.
910 
911  Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
912  have bitmask set but do not have ignoreMask set. Return the count.
913 
914  \param[in] mask: mask to define intersection region by.
915  \parma[in] footprint: footprint to define the intersection region by.
916  \param[in] bitmask: specific mask that we wish to count the number of occurances of.
917  \param[in] ignoreMask: pixels to not consider.
918  \return count of number of pixels in footprint with specified mask.
919  """
920  bbox = footprint.getBBox()
921  bbox.clip(mask.getBBox(afwImage.PARENT))
922  fp = afwImage.Mask(bbox)
923  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
924  footprint.spans.setMask(fp, bitmask)
925  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
926  (subMask.getArray() & ignoreMask) == 0).sum()
927 
928 
930  """!
931 \anchor SafeClipAssembleCoaddConfig
932 
933 \brief Configuration parameters for the SafeClipAssembleCoaddTask
934  """
935  clipDetection = pexConfig.ConfigurableField(
936  target=SourceDetectionTask,
937  doc="Detect sources on difference between unclipped and clipped coadd")
938  minClipFootOverlap = pexConfig.Field(
939  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
940  dtype=float,
941  default=0.6
942  )
943  minClipFootOverlapSingle = pexConfig.Field(
944  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
945  "clipped when only one visit overlaps",
946  dtype=float,
947  default=0.5
948  )
949  minClipFootOverlapDouble = pexConfig.Field(
950  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
951  "clipped when two visits overlap",
952  dtype=float,
953  default=0.45
954  )
955  maxClipFootOverlapDouble = pexConfig.Field(
956  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
957  "considering two visits",
958  dtype=float,
959  default=0.15
960  )
961  minBigOverlap = pexConfig.Field(
962  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
963  "when labeling clipped footprints",
964  dtype=int,
965  default=100
966  )
967 
968  def setDefaults(self):
969  # The numeric values for these configuration parameters were empirically determined, future work
970  # may further refine them.
971  AssembleCoaddConfig.setDefaults(self)
972  self.clipDetection.doTempLocalBackground = False
973  self.clipDetection.reEstimateBackground = False
974  self.clipDetection.returnOriginalFootprints = False
975  self.clipDetection.thresholdPolarity = "both"
976  self.clipDetection.thresholdValue = 2
977  self.clipDetection.nSigmaToGrow = 2
978  self.clipDetection.minPixels = 4
979  self.clipDetection.isotropicGrow = True
980  self.clipDetection.thresholdType = "pixel_stdev"
981  self.sigmaClip = 1.5
982  self.clipIter = 3
983  self.statistic = "MEAN"
984 
985  def validate(self):
986  if self.doSigmaClip:
987  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
988  "Ignoring doSigmaClip.")
989  self.doSigmaClip = False
990  if self.statistic != "MEAN":
991  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
992  "(%s chosen). Please set statistic to MEAN."
993  % (self.statistic))
994  AssembleCoaddTask.ConfigClass.validate(self)
995 
996 
997 ## \addtogroup LSST_task_documentation
998 ## \{
999 ## \page SafeClipAssembleCoaddTask
1000 ## \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask"
1001 ## \copybrief SafeClipAssembleCoaddTask
1002 ## \}
1003 
1004 
1006  """!
1007  \anchor SafeClipAssembleCoaddTask_
1008 
1009  \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
1010  with potential artifacts.
1011 
1012  \section pipe_tasks_assembleCoadd_Contents Contents
1013  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
1014  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
1015  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
1016  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
1017  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
1018  - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
1019 
1020  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
1021 
1022  \copybrief SafeClipAssembleCoaddTask
1023 
1024  Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
1025  SafeClipAssembleCoaddTask subtasks that task.
1026  In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1027  outliers).
1028  The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1029  PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1030  In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
1031  We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
1032  that there is an outlier and ii. this outlier appears on only one or two images.
1033  Such regions will not contribute to the final coadd.
1034  Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1035  Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1036  Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1037  surveys.
1038 
1039  SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1040  \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1041  \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1042 
1043  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1044  \copydoc \_\_init\_\_
1045 
1046  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1047  \copydoc run
1048 
1049  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1050  See \ref SafeClipAssembleCoaddConfig
1051 
1052  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1053  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1054  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1055  files.
1056  SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1057  subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1058  for further information.
1059 
1060  \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1061 
1062  SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1063  The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1064  '--legacyCoadd'.
1065  Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1066  (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1067  with a list of coaddTempExps to attempt to coadd (specified using
1068  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1069  Only the coaddTempExps that cover the specified tract and patch will be coadded.
1070  A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1071  command line argument:
1072  \code
1073  assembleCoadd.py --help
1074  \endcode
1075  To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1076  will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1077  begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1078  packages.
1079  This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1080  data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1081  <DL>
1082  <DT>processCcd</DT>
1083  <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1084  <DT>makeSkyMap</DT>
1085  <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1086  <DT>makeCoaddTempExp</DT>
1087  <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1088  </DL>
1089  We can perform all of these steps by running
1090  \code
1091  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1092  \endcode
1093  This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1094  as follows:
1095  \code
1096  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1097  \endcode
1098  This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1099  You may also choose to run:
1100  \code
1101  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1102  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1103  \endcode
1104  to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1105  discussed in \ref pipeTasks_multiBand.
1106  """
1107  ConfigClass = SafeClipAssembleCoaddConfig
1108  _DefaultName = "safeClipAssembleCoadd"
1109 
1110  def __init__(self, *args, **kwargs):
1111  """!
1112  \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1113  """
1114  AssembleCoaddTask.__init__(self, *args, **kwargs)
1115  schema = afwTable.SourceTable.makeMinimalSchema()
1116  self.makeSubtask("clipDetection", schema=schema)
1117 
1118  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1119  """!
1120  \brief Assemble the coadd for a region
1121 
1122  Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1123  that have outlier values in some individual visits. Detect clipped regions on the difference image and
1124  mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1125  overlap between the clipped region and a source.
1126  This leaves us with a set of footprints from the difference image that have been identified as having
1127  occured on just one or two individual visits. However, these footprints were generated from a
1128  difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1129  footprints acrosss the coadd difference in this process.
1130  Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1131  these are big footprints.
1132  Combine the small and big clipped footprints and mark them on a new bad mask plane
1133  Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1134  removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1135  bad mask plane.
1136 
1137  N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1138  parent task.
1139 
1140  @param skyInfo: Patch geometry information, from getSkyInfo
1141  @param tempExpRefList: List of data reference to tempExp
1142  @param imageScalerList: List of image scalers
1143  @param weightList: List of weights
1144  @param bgModelList: List of background models from background matching
1145  return coadd exposure
1146  """
1147  exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1148  mask = exp.getMaskedImage().getMask()
1149  mask.addMaskPlane("CLIPPED")
1150 
1151  result = self.detectClip(exp, tempExpRefList)
1152 
1153  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1154 
1155  # Go to individual visits for big footprints
1156  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1157  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1158  bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1159  maskClipValue, maskDetValue)
1160 
1161  # Create mask of the current clipped footprints
1162  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1163  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1164 
1165  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1166  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1167  maskClip |= maskClipBig
1168 
1169  # Assemble coadd from base class, but ignoring CLIPPED pixels
1170  badMaskPlanes = self.config.badMaskPlanes[:]
1171  badMaskPlanes.append("CLIPPED")
1172  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1173  coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1174  bgModelList, result.tempExpClipList, mask=badPixelMask)
1175 
1176  # Set the coadd CLIPPED mask from the footprints since currently pixels that are masked
1177  # do not get propagated
1178  maskExp = coaddExp.getMaskedImage().getMask()
1179  maskExp |= maskClip
1180 
1181  return coaddExp
1182 
1183  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList):
1184  """!
1185  \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1186 
1187  Generate a difference image between clipped and unclipped coadds.
1188  Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1189  Return the difference image.
1190 
1191  @param skyInfo: Patch geometry information, from getSkyInfo
1192  @param tempExpRefList: List of data reference to tempExp
1193  @param imageScalerList: List of image scalers
1194  @param weightList: List of weights
1195  @param bgModelList: List of background models from background matching
1196  @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1197  """
1198  # Clone and upcast self.config because current self.config is frozen
1199  config = AssembleCoaddConfig()
1200  # getattr necessary because subtasks do not survive Config.toDict()
1201  configIntersection = {k: getattr(self.config, k)
1202  for k, v in self.config.toDict().items() if (k in config.keys())}
1203  config.update(**configIntersection)
1204 
1205  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1206  config.statistic = 'MEAN'
1207  task = AssembleCoaddTask(config=config)
1208  coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1209 
1210  config.statistic = 'MEANCLIP'
1211  task = AssembleCoaddTask(config=config)
1212  coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1213 
1214  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1215  coaddDiff -= coaddClip.getMaskedImage()
1216  exp = afwImage.ExposureF(coaddDiff)
1217  exp.setPsf(coaddMean.getPsf())
1218  return exp
1219 
1220  def detectClip(self, exp, tempExpRefList):
1221  """!
1222  \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1223 
1224  Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1225  Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1226  fraction to thresholds set in the config.
1227  A different threshold is applied depending on the number of overlapping visits (restricted to one or
1228  two).
1229  If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1230  the coaddTempExp.
1231  Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1232  with the clipped footprints and a list of new masks for the coaddTempExps.
1233 
1234  \param[in] exp: Exposure to run detection on
1235  \param[in] tempExpRefList: List of data reference to tempExp
1236  \return struct containing:
1237  - clippedFootprints: list of clipped footprints
1238  - clippedIndices: indices for each clippedFootprint in tempExpRefList
1239  - tempExpClipList: list of new masks for tempExp
1240  """
1241  mask = exp.getMaskedImage().getMask()
1242  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1243  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1244  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1245  # Merge positive and negative together footprints together
1246  fpSet.positive.merge(fpSet.negative)
1247  footprints = fpSet.positive
1248  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1249  ignoreMask = self.getBadPixelMask()
1250 
1251  clipFootprints = []
1252  clipIndices = []
1253 
1254  # build a list with a mask for each visit which can be modified with clipping information
1255  tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.warpType),
1256  immediate=True).getMaskedImage().getMask() for
1257  tmpExpRef in tempExpRefList]
1258 
1259  for footprint in footprints.getFootprints():
1260  nPixel = footprint.getArea()
1261  overlap = [] # hold the overlap with each visit
1262  maskList = [] # which visit mask match
1263  indexList = [] # index of visit in global list
1264  for i, tmpExpMask in enumerate(tempExpClipList):
1265  # Determine the overlap with the footprint
1266  ignore = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1267  overlapDet = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1268  totPixel = nPixel - ignore
1269 
1270  # If we have more bad pixels than detection skip
1271  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1272  continue
1273  overlap.append(overlapDet/float(totPixel))
1274  maskList.append(tmpExpMask)
1275  indexList.append(i)
1276 
1277  overlap = numpy.array(overlap)
1278  if not len(overlap):
1279  continue
1280 
1281  keep = False # Should this footprint be marked as clipped?
1282  keepIndex = [] # Which tempExps does the clipped footprint belong to
1283 
1284  # If footprint only has one overlap use a lower threshold
1285  if len(overlap) == 1:
1286  if overlap[0] > self.config.minClipFootOverlapSingle:
1287  keep = True
1288  keepIndex = [0]
1289  else:
1290  # This is the general case where only visit should be clipped
1291  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1292  if len(clipIndex) == 1:
1293  keep = True
1294  keepIndex = [clipIndex[0]]
1295 
1296  # Test if there are clipped objects that overlap two different visits
1297  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1298  if len(clipIndex) == 2 and len(overlap) > 3:
1299  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1300  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1301  keep = True
1302  keepIndex = clipIndex
1303 
1304  if not keep:
1305  continue
1306 
1307  for index in keepIndex:
1308  footprint.spans.setMask(maskList[index], maskClipValue)
1309 
1310  clipIndices.append(numpy.array(indexList)[keepIndex])
1311  clipFootprints.append(footprint)
1312 
1313  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1314  tempExpClipList=tempExpClipList)
1315 
1316  def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1317  """!
1318  \brief Find footprints from individual tempExp footprints for large footprints.
1319 
1320  Identify big footprints composed of many sources in the coadd difference that may have originated in a
1321  large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1322  significantly with each source in all the coaddTempExps.
1323 
1324  \param[in] tempExpClipList: List of tempExp masks with clipping information
1325  \param[in] clipFootprints: List of clipped footprints
1326  \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1327  \param[in] maskClipValue: Mask value of clipped pixels
1328  \param[in] maskClipValue: Mask value of detected pixels
1329  \return list of big footprints
1330  """
1331  bigFootprintsCoadd = []
1332  ignoreMask = self.getBadPixelMask()
1333  for index, tmpExpMask in enumerate(tempExpClipList):
1334 
1335  # Create list of footprints from the DETECTED pixels
1336  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1337  afwImage.PARENT, True)
1338  maskVisitDet &= maskDetValue
1339  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1340 
1341  # build a mask of clipped footprints that are in this visit
1342  clippedFootprintsVisit = []
1343  for foot, clipIndex in zip(clipFootprints, clipIndices):
1344  if index not in clipIndex:
1345  continue
1346  clippedFootprintsVisit.append(foot)
1347  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1348  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1349 
1350  bigFootprintsVisit = []
1351  for foot in visitFootprints.getFootprints():
1352  if foot.getArea() < self.config.minBigOverlap:
1353  continue
1354  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1355  if nCount > self.config.minBigOverlap:
1356  bigFootprintsVisit.append(foot)
1357  bigFootprintsCoadd.append(foot)
1358 
1359  # Update single visit masks
1360  maskVisitClip.clearAllMaskPlanes()
1361  afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1362  tmpExpMask |= maskVisitClip
1363 
1364  return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of Warps.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip &amp; flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...