lsst.meas.deblender  14.0-1-g8b7e855+15
deblend.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy as np
24 
25 import lsst.pex.config as pexConf
26 import lsst.pipe.base as pipeBase
27 import lsst.afw.math as afwMath
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.geom.ellipses as afwEll
30 import lsst.afw.image as afwImage
31 import lsst.afw.detection as afwDet
32 import lsst.afw.table as afwTable
33 
34 __all__ = 'SourceDeblendConfig', 'SourceDeblendTask'
35 
36 
37 class SourceDeblendConfig(pexConf.Config):
38 
39  edgeHandling = pexConf.ChoiceField(
40  doc='What to do when a peak to be deblended is close to the edge of the image',
41  dtype=str, default='ramp',
42  allowed={
43  'clip': 'Clip the template at the edge AND the mirror of the edge.',
44  'ramp': 'Ramp down flux at the image edge by the PSF',
45  'noclip': 'Ignore the edge when building the symmetric template.',
46  }
47  )
48 
49  strayFluxToPointSources = pexConf.ChoiceField(
50  doc='When the deblender should attribute stray flux to point sources',
51  dtype=str, default='necessary',
52  allowed={
53  'necessary': 'When there is not an extended object in the footprint',
54  'always': 'Always',
55  'never': ('Never; stray flux will not be attributed to any deblended child '
56  'if the deblender thinks all peaks look like point sources'),
57  }
58  )
59 
60  assignStrayFlux = pexConf.Field(dtype=bool, default=True,
61  doc='Assign stray flux (not claimed by any child in the deblender) '
62  'to deblend children.')
63 
64  strayFluxRule = pexConf.ChoiceField(
65  doc='How to split flux among peaks',
66  dtype=str, default='trim',
67  allowed={
68  'r-to-peak': '~ 1/(1+R^2) to the peak',
69  'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. '
70  'CAUTION: this can be computationally expensive on large footprints!'),
71  'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka '
72  'Manhattan distance)'),
73  'trim': ('Shrink the parent footprint to pixels that are not assigned to children')
74  }
75  )
76 
77  clipStrayFluxFraction = pexConf.Field(dtype=float, default=0.001,
78  doc=('When splitting stray flux, clip fractions below '
79  'this value to zero.'))
80  psfChisq1 = pexConf.Field(dtype=float, default=1.5, optional=False,
81  doc=('Chi-squared per DOF cut for deciding a source is '
82  'a PSF during deblending (un-shifted PSF model)'))
83  psfChisq2 = pexConf.Field(dtype=float, default=1.5, optional=False,
84  doc=('Chi-squared per DOF cut for deciding a source is '
85  'PSF during deblending (shifted PSF model)'))
86  psfChisq2b = pexConf.Field(dtype=float, default=1.5, optional=False,
87  doc=('Chi-squared per DOF cut for deciding a source is '
88  'a PSF during deblending (shifted PSF model #2)'))
89  maxNumberOfPeaks = pexConf.Field(dtype=int, default=0,
90  doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
91  " (<= 0: unlimited)"))
92  maxFootprintArea = pexConf.Field(dtype=int, default=1000000,
93  doc=("Maximum area for footprints before they are ignored as large; "
94  "non-positive means no threshold applied"))
95  maxFootprintSize = pexConf.Field(dtype=int, default=0,
96  doc=("Maximum linear dimension for footprints before they are ignored "
97  "as large; non-positive means no threshold applied"))
98  minFootprintAxisRatio = pexConf.Field(dtype=float, default=0.0,
99  doc=("Minimum axis ratio for footprints before they are ignored "
100  "as large; non-positive means no threshold applied"))
101  notDeblendedMask = pexConf.Field(dtype=str, default="NOT_DEBLENDED", optional=True,
102  doc="Mask name for footprints not deblended, or None")
103 
104  tinyFootprintSize = pexConf.RangeField(dtype=int, default=2, min=2, inclusiveMin=True,
105  doc=('Footprints smaller in width or height than this value will '
106  'be ignored; minimum of 2 due to PSF gradient calculation.'))
107 
108  propagateAllPeaks = pexConf.Field(dtype=bool, default=False,
109  doc=('Guarantee that all peaks produce a child source.'))
110  catchFailures = pexConf.Field(dtype=bool, default=False,
111  doc=("If True, catch exceptions thrown by the deblender, log them, "
112  "and set a flag on the parent, instead of letting them propagate up"))
113  maskPlanes = pexConf.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
114  doc="Mask planes to ignore when performing statistics")
115  maskLimits = pexConf.DictField(
116  keytype=str,
117  itemtype=float,
118  default={},
119  doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
120  "Sources violating this limit will not be deblended."),
121  )
122  weightTemplates = pexConf.Field(dtype=bool, default=False,
123  doc=("If true, a least-squares fit of the templates will be done to the "
124  "full image. The templates will be re-weighted based on this fit."))
125  removeDegenerateTemplates = pexConf.Field(dtype=bool, default=False,
126  doc=("Try to remove similar templates?"))
127  maxTempDotProd = pexConf.Field(dtype=float, default=0.5,
128  doc=("If the dot product between two templates is larger than this value"
129  ", we consider them to be describing the same object (i.e. they are "
130  "degenerate). If one of the objects has been labeled as a PSF it "
131  "will be removed, otherwise the template with the lowest value will "
132  "be removed."))
133  medianSmoothTemplate = pexConf.Field(dtype=bool, default=True,
134  doc="Apply a smoothing filter to all of the template images")
135 
136 
142 
143 
144 class SourceDeblendTask(pipeBase.Task):
145  """!
146  \anchor SourceDeblendTask_
147 
148  \brief Split blended sources into individual sources.
149 
150  This task has no return value; it only modifies the SourceCatalog in-place.
151  """
152  ConfigClass = SourceDeblendConfig
153  _DefaultName = "sourceDeblend"
154 
155  def __init__(self, schema, peakSchema=None, **kwargs):
156  """!
157  Create the task, adding necessary fields to the given schema.
158 
159  @param[in,out] schema Schema object for measurement fields; will be modified in-place.
160  @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender.
161  Any fields beyond the PeakTable minimal schema will be transferred
162  to the main source Schema. If None, no fields will be transferred
163  from the Peaks.
164  @param[in] **kwargs Passed to Task.__init__.
165  """
166  pipeBase.Task.__init__(self, **kwargs)
167  peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
168  if peakSchema is None:
169  # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one
170  # to simplify downstream code
171  self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
172  else:
173  self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
174  for item in peakSchema:
175  if item.key not in peakMinimalSchema:
176  self.peakSchemaMapper.addMapping(item.key, item.field)
177  # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't
178  # updating this Schema in place. That's probably a design flaw, but in the meantime,
179  # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually,
180  # by adding the same fields to both.
181  schema.addField(item.field)
182  assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
183  self.addSchemaKeys(schema)
184 
185  def addSchemaKeys(self, schema):
186  self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
187  doc='Number of children this object has (defaults to 0)')
188  self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
189  doc='Deblender thought this source looked like a PSF')
190  self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter',
191  'If deblended-as-psf, the PSF centroid', "pixel")
192  self.psfFluxKey = schema.addField('deblend_psfFlux', type='D',
193  doc='If deblended-as-psf, the PSF flux')
194  self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
195  doc='Source had too many peaks; '
196  'only the brightest were included')
197  self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
198  doc='Parent footprint covered too many pixels')
199  self.maskedKey = schema.addField('deblend_masked', type='Flag',
200  doc='Parent footprint was predominantly masked')
201 
202  if self.config.catchFailures:
203  self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
204  doc="Deblending failed on source")
205 
206  self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
207  doc="Deblender skipped this source")
208 
209  self.deblendRampedTemplateKey = schema.addField(
210  'deblend_rampedTemplate', type='Flag',
211  doc=('This source was near an image edge and the deblender used '
212  '"ramp" edge-handling.'))
213 
214  self.deblendPatchedTemplateKey = schema.addField(
215  'deblend_patchedTemplate', type='Flag',
216  doc=('This source was near an image edge and the deblender used '
217  '"patched" edge-handling.'))
218 
219  self.hasStrayFluxKey = schema.addField(
220  'deblend_hasStrayFlux', type='Flag',
221  doc=('This source was assigned some stray flux'))
222 
223  self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in (
224  self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey,
225  self.tooManyPeaksKey, self.tooBigKey)))
226 
227  @pipeBase.timeMethod
228  def run(self, exposure, sources):
229  """!
230  Get the psf from the provided exposure and then run deblend().
231 
232  @param[in] exposure Exposure to process
233  @param[in,out] sources SourceCatalog containing sources detected on this exposure.
234 
235  @return None
236  """
237  psf = exposure.getPsf()
238  self.deblend(exposure, sources, psf)
239 
240  def _getPsfFwhm(self, psf, bbox):
241  # It should be easier to get a PSF's fwhm;
242  # https://dev.lsstcorp.org/trac/ticket/3030
243  return psf.computeShape().getDeterminantRadius() * 2.35
244 
245  @pipeBase.timeMethod
246  def deblend(self, exposure, srcs, psf):
247  """!
248  Deblend.
249 
250  @param[in] exposure Exposure to process
251  @param[in,out] srcs SourceCatalog containing sources detected on this exposure.
252  @param[in] psf PSF
253 
254  @return None
255  """
256  self.log.info("Deblending %d sources" % len(srcs))
257 
258  from lsst.meas.deblender.baseline import deblend
259 
260  # find the median stdev in the image...
261  mi = exposure.getMaskedImage()
262  statsCtrl = afwMath.StatisticsControl()
263  statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
264  stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
265  sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
266  self.log.trace('sigma1: %g', sigma1)
267 
268  n0 = len(srcs)
269  nparents = 0
270  for i, src in enumerate(srcs):
271  #t0 = time.clock()
272 
273  fp = src.getFootprint()
274  pks = fp.getPeaks()
275 
276  # Since we use the first peak for the parent object, we should propagate its flags
277  # to the parent source.
278  src.assign(pks[0], self.peakSchemaMapper)
279 
280  if len(pks) < 2:
281  continue
282 
283  if self.isLargeFootprint(fp):
284  src.set(self.tooBigKey, True)
285  self.skipParent(src, mi.getMask())
286  self.log.trace('Parent %i: skipping large footprint', int(src.getId()))
287  continue
288  if self.isMasked(fp, exposure.getMaskedImage().getMask()):
289  src.set(self.maskedKey, True)
290  self.skipParent(src, mi.getMask())
291  self.log.trace('Parent %i: skipping masked footprint', int(src.getId()))
292  continue
293 
294  nparents += 1
295  bb = fp.getBBox()
296  psf_fwhm = self._getPsfFwhm(psf, bb)
297 
298  self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks))
299 
300  self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
301  npre = len(srcs)
302 
303  # This should really be set in deblend, but deblend doesn't have access to the src
304  src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
305 
306  try:
307  res = deblend(
308  fp, mi, psf, psf_fwhm, sigma1=sigma1,
309  psfChisqCut1=self.config.psfChisq1,
310  psfChisqCut2=self.config.psfChisq2,
311  psfChisqCut2b=self.config.psfChisq2b,
312  maxNumberOfPeaks=self.config.maxNumberOfPeaks,
313  strayFluxToPointSources=self.config.strayFluxToPointSources,
314  assignStrayFlux=self.config.assignStrayFlux,
315  strayFluxAssignment=self.config.strayFluxRule,
316  rampFluxAtEdge=(self.config.edgeHandling == 'ramp'),
317  patchEdges=(self.config.edgeHandling == 'noclip'),
318  tinyFootprintSize=self.config.tinyFootprintSize,
319  clipStrayFluxFraction=self.config.clipStrayFluxFraction,
320  weightTemplates=self.config.weightTemplates,
321  removeDegenerateTemplates=self.config.removeDegenerateTemplates,
322  maxTempDotProd=self.config.maxTempDotProd,
323  medianSmoothTemplate=self.config.medianSmoothTemplate
324  )
325  if self.config.catchFailures:
326  src.set(self.deblendFailedKey, False)
327  except Exception as e:
328  if self.config.catchFailures:
329  self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e))
330  src.set(self.deblendFailedKey, True)
331  import traceback
332  traceback.print_exc()
333  continue
334  else:
335  raise
336 
337  kids = []
338  nchild = 0
339  for j, peak in enumerate(res.deblendedParents[0].peaks):
340  heavy = peak.getFluxPortion()
341  if heavy is None or peak.skip:
342  src.set(self.deblendSkippedKey, True)
343  if not self.config.propagateAllPeaks:
344  # Don't care
345  continue
346  # We need to preserve the peak: make sure we have enough info to create a minimal
347  # child src
348  self.log.trace("Peak at (%i,%i) failed. Using minimal default info for child.",
349  pks[j].getIx(), pks[j].getIy())
350  if heavy is None:
351  # copy the full footprint and strip out extra peaks
352  foot = afwDet.Footprint(src.getFootprint())
353  peakList = foot.getPeaks()
354  peakList.clear()
355  peakList.append(peak.peak)
356  zeroMimg = afwImage.MaskedImageF(foot.getBBox())
357  heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
358  if peak.deblendedAsPsf:
359  if peak.psfFitFlux is None:
360  peak.psfFitFlux = 0.0
361  if peak.psfFitCenter is None:
362  peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
363 
364  assert(len(heavy.getPeaks()) == 1)
365 
366  src.set(self.deblendSkippedKey, False)
367  child = srcs.addNew()
368  nchild += 1
369  child.assign(heavy.getPeaks()[0], self.peakSchemaMapper)
370  child.setParent(src.getId())
371  child.setFootprint(heavy)
372  child.set(self.psfKey, peak.deblendedAsPsf)
373  child.set(self.hasStrayFluxKey, peak.strayFlux is not None)
374  if peak.deblendedAsPsf:
375  (cx, cy) = peak.psfFitCenter
376  child.set(self.psfCenterKey, afwGeom.Point2D(cx, cy))
377  child.set(self.psfFluxKey, peak.psfFitFlux)
378  child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate)
379  child.set(self.deblendPatchedTemplateKey, peak.patched)
380  kids.append(child)
381 
382  # Child footprints may extend beyond the full extent of their parent's which
383  # results in a failure of the replace-by-noise code to reinstate these pixels
384  # to their original values. The following updates the parent footprint
385  # in-place to ensure it contains the full union of itself and all of its
386  # children's footprints.
387  spans = src.getFootprint().spans
388  for child in kids:
389  spans = spans.union(child.getFootprint().spans)
390  src.getFootprint().setSpans(spans)
391 
392  src.set(self.nChildKey, nchild)
393 
394  self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
395  #print 'Deblending parent id', src.getId(), 'took', time.clock() - t0
396 
397  n1 = len(srcs)
398  self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
399  % (n0, nparents, n1-n0, n1))
400 
401  def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1):
402  pass
403 
404  def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
405  pass
406 
407  def isLargeFootprint(self, footprint):
408  """Returns whether a Footprint is large
409 
410  'Large' is defined by thresholds on the area, size and axis ratio.
411  These may be disabled independently by configuring them to be non-positive.
412 
413  This is principally intended to get rid of satellite streaks, which the
414  deblender or other downstream processing can have trouble dealing with
415  (e.g., multiple large HeavyFootprints can chew up memory).
416  """
417  if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
418  return True
419  if self.config.maxFootprintSize > 0:
420  bbox = footprint.getBBox()
421  if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
422  return True
423  if self.config.minFootprintAxisRatio > 0:
424  axes = afwEll.Axes(footprint.getShape())
425  if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
426  return True
427  return False
428 
429  def isMasked(self, footprint, mask):
430  """Returns whether the footprint violates the mask limits"""
431  size = float(footprint.getArea())
432  for maskName, limit in self.config.maskLimits.items():
433  maskVal = mask.getPlaneBitMask(maskName)
434  unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels
435  if (size - unmaskedSpan.getArea())/size > limit:
436  return True
437  return False
438 
439  def skipParent(self, source, mask):
440  """Indicate that the parent source is not being deblended
441 
442  We set the appropriate flags and mask.
443 
444  @param source The source to flag as skipped
445  @param mask The mask to update
446  """
447  fp = source.getFootprint()
448  source.set(self.deblendSkippedKey, True)
449  source.set(self.nChildKey, len(fp.getPeaks())) # It would have this many if we deblended them all
450  if self.config.notDeblendedMask:
451  mask.addMaskPlane(self.config.notDeblendedMask)
452  fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
Definition: deblend.py:404
def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
Definition: deblend.py:401
def run(self, exposure, sources)
Get the psf from the provided exposure and then run deblend().
Definition: deblend.py:228
Split blended sources into individual sources.
Definition: deblend.py:144
def deblend(self, exposure, srcs, psf)
Deblend.
Definition: deblend.py:246
def __init__(self, schema, peakSchema=None, kwargs)
Create the task, adding necessary fields to the given schema.
Definition: deblend.py:155
def isMasked(self, footprint, mask)
Definition: deblend.py:429