Coverage for python/lsst/meas/deblender/sourceDeblendTask.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_deblender.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy as np
25import lsst.log
26import lsst.pex.config as pexConfig
27import lsst.pipe.base as pipeBase
28import lsst.afw.math as afwMath
29import lsst.geom as geom
30import lsst.afw.geom.ellipses as afwEll
31import lsst.afw.image as afwImage
32import lsst.afw.detection as afwDet
33import lsst.afw.table as afwTable
35logger = lsst.log.Log.getLogger("meas.deblender.deblend")
37__all__ = 'SourceDeblendConfig', 'SourceDeblendTask'
40class SourceDeblendConfig(pexConfig.Config):
42 edgeHandling = pexConfig.ChoiceField(
43 doc='What to do when a peak to be deblended is close to the edge of the image',
44 dtype=str, default='ramp',
45 allowed={
46 'clip': 'Clip the template at the edge AND the mirror of the edge.',
47 'ramp': 'Ramp down flux at the image edge by the PSF',
48 'noclip': 'Ignore the edge when building the symmetric template.',
49 }
50 )
52 strayFluxToPointSources = pexConfig.ChoiceField(
53 doc='When the deblender should attribute stray flux to point sources',
54 dtype=str, default='necessary',
55 allowed={
56 'necessary': 'When there is not an extended object in the footprint',
57 'always': 'Always',
58 'never': ('Never; stray flux will not be attributed to any deblended child '
59 'if the deblender thinks all peaks look like point sources'),
60 }
61 )
63 assignStrayFlux = pexConfig.Field(dtype=bool, default=True,
64 doc='Assign stray flux (not claimed by any child in the deblender) '
65 'to deblend children.')
67 strayFluxRule = pexConfig.ChoiceField(
68 doc='How to split flux among peaks',
69 dtype=str, default='trim',
70 allowed={
71 'r-to-peak': '~ 1/(1+R^2) to the peak',
72 'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. '
73 'CAUTION: this can be computationally expensive on large footprints!'),
74 'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka '
75 'Manhattan distance)'),
76 'trim': ('Shrink the parent footprint to pixels that are not assigned to children')
77 }
78 )
80 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
81 doc=('When splitting stray flux, clip fractions below '
82 'this value to zero.'))
83 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=False,
84 doc=('Chi-squared per DOF cut for deciding a source is '
85 'a PSF during deblending (un-shifted PSF model)'))
86 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=False,
87 doc=('Chi-squared per DOF cut for deciding a source is '
88 'PSF during deblending (shifted PSF model)'))
89 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=False,
90 doc=('Chi-squared per DOF cut for deciding a source is '
91 'a PSF during deblending (shifted PSF model #2)'))
92 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0,
93 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
94 " (<= 0: unlimited)"))
95 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000,
96 doc=("Maximum area for footprints before they are ignored as large; "
97 "non-positive means no threshold applied"))
98 maxFootprintSize = pexConfig.Field(dtype=int, default=0,
99 doc=("Maximum linear dimension for footprints before they are ignored "
100 "as large; non-positive means no threshold applied"))
101 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0,
102 doc=("Minimum axis ratio for footprints before they are ignored "
103 "as large; non-positive means no threshold applied"))
104 notDeblendedMask = pexConfig.Field(dtype=str, default="NOT_DEBLENDED", optional=True,
105 doc="Mask name for footprints not deblended, or None")
107 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=True,
108 doc=('Footprints smaller in width or height than this value '
109 'will be ignored; minimum of 2 due to PSF gradient '
110 'calculation.'))
112 propagateAllPeaks = pexConfig.Field(dtype=bool, default=False,
113 doc=('Guarantee that all peaks produce a child source.'))
114 catchFailures = pexConfig.Field(
115 dtype=bool, default=False,
116 doc=("If True, catch exceptions thrown by the deblender, log them, "
117 "and set a flag on the parent, instead of letting them propagate up"))
118 maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
119 doc="Mask planes to ignore when performing statistics")
120 maskLimits = pexConfig.DictField(
121 keytype=str,
122 itemtype=float,
123 default={},
124 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
125 "Sources violating this limit will not be deblended."),
126 )
127 weightTemplates = pexConfig.Field(
128 dtype=bool, default=False,
129 doc=("If true, a least-squares fit of the templates will be done to the "
130 "full image. The templates will be re-weighted based on this fit."))
131 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=False,
132 doc=("Try to remove similar templates?"))
133 maxTempDotProd = pexConfig.Field(
134 dtype=float, default=0.5,
135 doc=("If the dot product between two templates is larger than this value, we consider them to be "
136 "describing the same object (i.e. they are degenerate). If one of the objects has been "
137 "labeled as a PSF it will be removed, otherwise the template with the lowest value will "
138 "be removed."))
139 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=True,
140 doc="Apply a smoothing filter to all of the template images")
142## \addtogroup LSST_task_documentation
143## \{
144## \page SourceDeblendTask
145## \ref SourceDeblendTask_ "SourceDeblendTask"
146## \copybrief SourceDeblendTask
147## \}
150class SourceDeblendTask(pipeBase.Task):
151 """!
152 \anchor SourceDeblendTask_
154 \brief Split blended sources into individual sources.
156 This task has no return value; it only modifies the SourceCatalog in-place.
157 """
158 ConfigClass = SourceDeblendConfig
159 _DefaultName = "sourceDeblend"
161 def __init__(self, schema, peakSchema=None, **kwargs):
162 """!
163 Create the task, adding necessary fields to the given schema.
165 @param[in,out] schema Schema object for measurement fields; will be modified in-place.
166 @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender.
167 Any fields beyond the PeakTable minimal schema will be transferred
168 to the main source Schema. If None, no fields will be transferred
169 from the Peaks.
170 @param[in] **kwargs Passed to Task.__init__.
171 """
172 pipeBase.Task.__init__(self, **kwargs)
173 self.schema = schema
174 self.toCopyFromParent = [item.key for item in self.schema
175 if item.field.getName().startswith("merge_footprint")]
176 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
177 if peakSchema is None:
178 # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one
179 # to simplify downstream code
180 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
181 else:
182 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
183 for item in peakSchema:
184 if item.key not in peakMinimalSchema:
185 self.peakSchemaMapper.addMapping(item.key, item.field)
186 # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't
187 # updating this Schema in place. That's probably a design flaw, but in the meantime,
188 # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually,
189 # by adding the same fields to both.
190 schema.addField(item.field)
191 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
192 self.addSchemaKeys(schema)
194 def addSchemaKeys(self, schema):
195 self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
196 doc='Number of children this object has (defaults to 0)')
197 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
198 doc='Deblender thought this source looked like a PSF')
199 self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter',
200 'If deblended-as-psf, the PSF centroid', "pixel")
201 self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D',
202 doc='If deblended-as-psf, the instrumental PSF flux', units='count')
203 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
204 doc='Source had too many peaks; '
205 'only the brightest were included')
206 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
207 doc='Parent footprint covered too many pixels')
208 self.maskedKey = schema.addField('deblend_masked', type='Flag',
209 doc='Parent footprint was predominantly masked')
211 if self.config.catchFailures:
212 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
213 doc="Deblending failed on source")
215 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
216 doc="Deblender skipped this source")
218 self.deblendRampedTemplateKey = schema.addField(
219 'deblend_rampedTemplate', type='Flag',
220 doc=('This source was near an image edge and the deblender used '
221 '"ramp" edge-handling.'))
223 self.deblendPatchedTemplateKey = schema.addField(
224 'deblend_patchedTemplate', type='Flag',
225 doc=('This source was near an image edge and the deblender used '
226 '"patched" edge-handling.'))
228 self.hasStrayFluxKey = schema.addField(
229 'deblend_hasStrayFlux', type='Flag',
230 doc=('This source was assigned some stray flux'))
232 self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in (
233 self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey,
234 self.tooManyPeaksKey, self.tooBigKey)))
236 @pipeBase.timeMethod
237 def run(self, exposure, sources):
238 """!
239 Get the psf from the provided exposure and then run deblend().
241 @param[in] exposure Exposure to process
242 @param[in,out] sources SourceCatalog containing sources detected on this exposure.
244 @return None
245 """
246 psf = exposure.getPsf()
247 assert sources.getSchema() == self.schema
248 self.deblend(exposure, sources, psf)
250 def _getPsfFwhm(self, psf, bbox):
251 # It should be easier to get a PSF's fwhm;
252 # https://dev.lsstcorp.org/trac/ticket/3030
253 return psf.computeShape().getDeterminantRadius() * 2.35
255 @pipeBase.timeMethod
256 def deblend(self, exposure, srcs, psf):
257 """!
258 Deblend.
260 @param[in] exposure Exposure to process
261 @param[in,out] srcs SourceCatalog containing sources detected on this exposure.
262 @param[in] psf PSF
264 @return None
265 """
266 self.log.info("Deblending %d sources" % len(srcs))
268 from lsst.meas.deblender.baseline import deblend
270 # find the median stdev in the image...
271 mi = exposure.getMaskedImage()
272 statsCtrl = afwMath.StatisticsControl()
273 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
274 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
275 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
276 self.log.trace('sigma1: %g', sigma1)
278 n0 = len(srcs)
279 nparents = 0
280 for i, src in enumerate(srcs):
281 # t0 = time.clock()
283 fp = src.getFootprint()
284 pks = fp.getPeaks()
286 # Since we use the first peak for the parent object, we should propagate its flags
287 # to the parent source.
288 src.assign(pks[0], self.peakSchemaMapper)
290 if len(pks) < 2:
291 continue
293 if self.isLargeFootprint(fp):
294 src.set(self.tooBigKey, True)
295 self.skipParent(src, mi.getMask())
296 self.log.warn('Parent %i: skipping large footprint (area: %i)',
297 int(src.getId()), int(fp.getArea()))
298 continue
299 if self.isMasked(fp, exposure.getMaskedImage().getMask()):
300 src.set(self.maskedKey, True)
301 self.skipParent(src, mi.getMask())
302 self.log.warn('Parent %i: skipping masked footprint (area: %i)',
303 int(src.getId()), int(fp.getArea()))
304 continue
306 nparents += 1
307 bb = fp.getBBox()
308 psf_fwhm = self._getPsfFwhm(psf, bb)
310 self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks))
312 self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
313 npre = len(srcs)
315 # This should really be set in deblend, but deblend doesn't have access to the src
316 src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
318 try:
319 res = deblend(
320 fp, mi, psf, psf_fwhm, sigma1=sigma1,
321 psfChisqCut1=self.config.psfChisq1,
322 psfChisqCut2=self.config.psfChisq2,
323 psfChisqCut2b=self.config.psfChisq2b,
324 maxNumberOfPeaks=self.config.maxNumberOfPeaks,
325 strayFluxToPointSources=self.config.strayFluxToPointSources,
326 assignStrayFlux=self.config.assignStrayFlux,
327 strayFluxAssignment=self.config.strayFluxRule,
328 rampFluxAtEdge=(self.config.edgeHandling == 'ramp'),
329 patchEdges=(self.config.edgeHandling == 'noclip'),
330 tinyFootprintSize=self.config.tinyFootprintSize,
331 clipStrayFluxFraction=self.config.clipStrayFluxFraction,
332 weightTemplates=self.config.weightTemplates,
333 removeDegenerateTemplates=self.config.removeDegenerateTemplates,
334 maxTempDotProd=self.config.maxTempDotProd,
335 medianSmoothTemplate=self.config.medianSmoothTemplate
336 )
337 if self.config.catchFailures:
338 src.set(self.deblendFailedKey, False)
339 except Exception as e:
340 if self.config.catchFailures:
341 self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e))
342 src.set(self.deblendFailedKey, True)
343 import traceback
344 traceback.print_exc()
345 continue
346 else:
347 raise
349 kids = []
350 nchild = 0
351 for j, peak in enumerate(res.deblendedParents[0].peaks):
352 heavy = peak.getFluxPortion()
353 if heavy is None or peak.skip:
354 src.set(self.deblendSkippedKey, True)
355 if not self.config.propagateAllPeaks:
356 # Don't care
357 continue
358 # We need to preserve the peak: make sure we have enough info to create a minimal
359 # child src
360 self.log.trace("Peak at (%i,%i) failed. Using minimal default info for child.",
361 pks[j].getIx(), pks[j].getIy())
362 if heavy is None:
363 # copy the full footprint and strip out extra peaks
364 foot = afwDet.Footprint(src.getFootprint())
365 peakList = foot.getPeaks()
366 peakList.clear()
367 peakList.append(peak.peak)
368 zeroMimg = afwImage.MaskedImageF(foot.getBBox())
369 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
370 if peak.deblendedAsPsf:
371 if peak.psfFitFlux is None:
372 peak.psfFitFlux = 0.0
373 if peak.psfFitCenter is None:
374 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
376 assert(len(heavy.getPeaks()) == 1)
378 src.set(self.deblendSkippedKey, False)
379 child = srcs.addNew()
380 nchild += 1
381 for key in self.toCopyFromParent:
382 child.set(key, src.get(key))
383 child.assign(heavy.getPeaks()[0], self.peakSchemaMapper)
384 child.setParent(src.getId())
385 child.setFootprint(heavy)
386 child.set(self.psfKey, peak.deblendedAsPsf)
387 child.set(self.hasStrayFluxKey, peak.strayFlux is not None)
388 if peak.deblendedAsPsf:
389 (cx, cy) = peak.psfFitCenter
390 child.set(self.psfCenterKey, geom.Point2D(cx, cy))
391 child.set(self.psfFluxKey, peak.psfFitFlux)
392 child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate)
393 child.set(self.deblendPatchedTemplateKey, peak.patched)
394 kids.append(child)
396 # Child footprints may extend beyond the full extent of their parent's which
397 # results in a failure of the replace-by-noise code to reinstate these pixels
398 # to their original values. The following updates the parent footprint
399 # in-place to ensure it contains the full union of itself and all of its
400 # children's footprints.
401 spans = src.getFootprint().spans
402 for child in kids:
403 spans = spans.union(child.getFootprint().spans)
404 src.getFootprint().setSpans(spans)
406 src.set(self.nChildKey, nchild)
408 self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
409 # print('Deblending parent id', src.getId(), 'took', time.clock() - t0)
411 n1 = len(srcs)
412 self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
413 % (n0, nparents, n1-n0, n1))
415 def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1):
416 pass
418 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
419 pass
421 def isLargeFootprint(self, footprint):
422 """Returns whether a Footprint is large
424 'Large' is defined by thresholds on the area, size and axis ratio.
425 These may be disabled independently by configuring them to be non-positive.
427 This is principally intended to get rid of satellite streaks, which the
428 deblender or other downstream processing can have trouble dealing with
429 (e.g., multiple large HeavyFootprints can chew up memory).
430 """
431 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
432 return True
433 if self.config.maxFootprintSize > 0:
434 bbox = footprint.getBBox()
435 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
436 return True
437 if self.config.minFootprintAxisRatio > 0:
438 axes = afwEll.Axes(footprint.getShape())
439 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
440 return True
441 return False
443 def isMasked(self, footprint, mask):
444 """Returns whether the footprint violates the mask limits"""
445 size = float(footprint.getArea())
446 for maskName, limit in self.config.maskLimits.items():
447 maskVal = mask.getPlaneBitMask(maskName)
448 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels
449 if (size - unmaskedSpan.getArea())/size > limit:
450 return True
451 return False
453 def skipParent(self, source, mask):
454 """Indicate that the parent source is not being deblended
456 We set the appropriate flags and mask.
458 @param source The source to flag as skipped
459 @param mask The mask to update
460 """
461 fp = source.getFootprint()
462 source.set(self.deblendSkippedKey, True)
463 source.set(self.nChildKey, len(fp.getPeaks())) # It would have this many if we deblended them all
464 if self.config.notDeblendedMask:
465 mask.addMaskPlane(self.config.notDeblendedMask)
466 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))