Coverage for python/lsst/meas/deblender/sourceDeblendTask.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_deblender.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy as np
25import lsst.log
26import lsst.pex.config as pexConfig
27import lsst.pipe.base as pipeBase
28import lsst.afw.math as afwMath
29import lsst.geom as geom
30import lsst.afw.geom.ellipses as afwEll
31import lsst.afw.image as afwImage
32import lsst.afw.detection as afwDet
33import lsst.afw.table as afwTable
35logger = lsst.log.Log.getLogger("meas.deblender.deblend")
37__all__ = 'SourceDeblendConfig', 'SourceDeblendTask'
40class SourceDeblendConfig(pexConfig.Config):
42 edgeHandling = pexConfig.ChoiceField(
43 doc='What to do when a peak to be deblended is close to the edge of the image',
44 dtype=str, default='ramp',
45 allowed={
46 'clip': 'Clip the template at the edge AND the mirror of the edge.',
47 'ramp': 'Ramp down flux at the image edge by the PSF',
48 'noclip': 'Ignore the edge when building the symmetric template.',
49 }
50 )
52 strayFluxToPointSources = pexConfig.ChoiceField(
53 doc='When the deblender should attribute stray flux to point sources',
54 dtype=str, default='necessary',
55 allowed={
56 'necessary': 'When there is not an extended object in the footprint',
57 'always': 'Always',
58 'never': ('Never; stray flux will not be attributed to any deblended child '
59 'if the deblender thinks all peaks look like point sources'),
60 }
61 )
63 assignStrayFlux = pexConfig.Field(dtype=bool, default=True,
64 doc='Assign stray flux (not claimed by any child in the deblender) '
65 'to deblend children.')
67 strayFluxRule = pexConfig.ChoiceField(
68 doc='How to split flux among peaks',
69 dtype=str, default='trim',
70 allowed={
71 'r-to-peak': '~ 1/(1+R^2) to the peak',
72 'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. '
73 'CAUTION: this can be computationally expensive on large footprints!'),
74 'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka '
75 'Manhattan distance)'),
76 'trim': ('Shrink the parent footprint to pixels that are not assigned to children')
77 }
78 )
80 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
81 doc=('When splitting stray flux, clip fractions below '
82 'this value to zero.'))
83 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=False,
84 doc=('Chi-squared per DOF cut for deciding a source is '
85 'a PSF during deblending (un-shifted PSF model)'))
86 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=False,
87 doc=('Chi-squared per DOF cut for deciding a source is '
88 'PSF during deblending (shifted PSF model)'))
89 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=False,
90 doc=('Chi-squared per DOF cut for deciding a source is '
91 'a PSF during deblending (shifted PSF model #2)'))
92 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0,
93 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
94 " (<= 0: unlimited)"))
95 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000,
96 doc=("Maximum area for footprints before they are ignored as large; "
97 "non-positive means no threshold applied"))
98 maxFootprintSize = pexConfig.Field(dtype=int, default=0,
99 doc=("Maximum linear dimension for footprints before they are ignored "
100 "as large; non-positive means no threshold applied"))
101 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0,
102 doc=("Minimum axis ratio for footprints before they are ignored "
103 "as large; non-positive means no threshold applied"))
104 notDeblendedMask = pexConfig.Field(dtype=str, default="NOT_DEBLENDED", optional=True,
105 doc="Mask name for footprints not deblended, or None")
107 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=True,
108 doc=('Footprints smaller in width or height than this value '
109 'will be ignored; minimum of 2 due to PSF gradient '
110 'calculation.'))
112 propagateAllPeaks = pexConfig.Field(dtype=bool, default=False,
113 doc=('Guarantee that all peaks produce a child source.'))
114 catchFailures = pexConfig.Field(
115 dtype=bool, default=False,
116 doc=("If True, catch exceptions thrown by the deblender, log them, "
117 "and set a flag on the parent, instead of letting them propagate up"))
118 maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
119 doc="Mask planes to ignore when performing statistics")
120 maskLimits = pexConfig.DictField(
121 keytype=str,
122 itemtype=float,
123 default={},
124 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
125 "Sources violating this limit will not be deblended."),
126 )
127 weightTemplates = pexConfig.Field(
128 dtype=bool, default=False,
129 doc=("If true, a least-squares fit of the templates will be done to the "
130 "full image. The templates will be re-weighted based on this fit."))
131 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=False,
132 doc=("Try to remove similar templates?"))
133 maxTempDotProd = pexConfig.Field(
134 dtype=float, default=0.5,
135 doc=("If the dot product between two templates is larger than this value, we consider them to be "
136 "describing the same object (i.e. they are degenerate). If one of the objects has been "
137 "labeled as a PSF it will be removed, otherwise the template with the lowest value will "
138 "be removed."))
139 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=True,
140 doc="Apply a smoothing filter to all of the template images")
142## \addtogroup LSST_task_documentation
143## \{
144## \page SourceDeblendTask
145## \ref SourceDeblendTask_ "SourceDeblendTask"
146## \copybrief SourceDeblendTask
147## \}
150class SourceDeblendTask(pipeBase.Task):
151 """!
152 \anchor SourceDeblendTask_
154 \brief Split blended sources into individual sources.
156 This task has no return value; it only modifies the SourceCatalog in-place.
157 """
158 ConfigClass = SourceDeblendConfig
159 _DefaultName = "sourceDeblend"
161 def __init__(self, schema, peakSchema=None, **kwargs):
162 """!
163 Create the task, adding necessary fields to the given schema.
165 @param[in,out] schema Schema object for measurement fields; will be modified in-place.
166 @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender.
167 Any fields beyond the PeakTable minimal schema will be transferred
168 to the main source Schema. If None, no fields will be transferred
169 from the Peaks.
170 @param[in] **kwargs Passed to Task.__init__.
171 """
172 pipeBase.Task.__init__(self, **kwargs)
173 self.schema = schema
174 self.toCopyFromParent = [item.key for item in self.schema
175 if item.field.getName().startswith("merge_footprint")]
176 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
177 if peakSchema is None:
178 # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one
179 # to simplify downstream code
180 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
181 else:
182 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
183 for item in peakSchema:
184 if item.key not in peakMinimalSchema:
185 self.peakSchemaMapper.addMapping(item.key, item.field)
186 # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't
187 # updating this Schema in place. That's probably a design flaw, but in the meantime,
188 # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually,
189 # by adding the same fields to both.
190 schema.addField(item.field)
191 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
192 self.addSchemaKeys(schema)
194 def addSchemaKeys(self, schema):
195 self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
196 doc='Number of children this object has (defaults to 0)')
197 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
198 doc='Deblender thought this source looked like a PSF')
199 self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter',
200 'If deblended-as-psf, the PSF centroid', "pixel")
201 self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D',
202 doc='If deblended-as-psf, the instrumental PSF flux', units='count')
203 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
204 doc='Source had too many peaks; '
205 'only the brightest were included')
206 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
207 doc='Parent footprint covered too many pixels')
208 self.maskedKey = schema.addField('deblend_masked', type='Flag',
209 doc='Parent footprint was predominantly masked')
211 if self.config.catchFailures:
212 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
213 doc="Deblending failed on source")
215 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
216 doc="Deblender skipped this source")
218 self.deblendRampedTemplateKey = schema.addField(
219 'deblend_rampedTemplate', type='Flag',
220 doc=('This source was near an image edge and the deblender used '
221 '"ramp" edge-handling.'))
223 self.deblendPatchedTemplateKey = schema.addField(
224 'deblend_patchedTemplate', type='Flag',
225 doc=('This source was near an image edge and the deblender used '
226 '"patched" edge-handling.'))
228 self.hasStrayFluxKey = schema.addField(
229 'deblend_hasStrayFlux', type='Flag',
230 doc=('This source was assigned some stray flux'))
232 self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in (
233 self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey,
234 self.tooManyPeaksKey, self.tooBigKey)))
235 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center",
236 doc="Center used to apply constraints in scarlet",
237 unit="pixel")
238 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32,
239 doc="ID of the peak in the parent footprint. "
240 "This is not unique, but the combination of 'parent'"
241 "and 'peakId' should be for all child sources. "
242 "Top level blends with no parents have 'peakId=0'")
243 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32,
244 doc="Number of initial peaks in the blend. "
245 "This includes peaks that may have been culled "
246 "during deblending or failed to deblend")
247 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32,
248 doc="Same as deblend_n_peaks, but the number of peaks "
249 "in the parent footprint")
251 @pipeBase.timeMethod
252 def run(self, exposure, sources):
253 """!
254 Get the psf from the provided exposure and then run deblend().
256 @param[in] exposure Exposure to process
257 @param[in,out] sources SourceCatalog containing sources detected on this exposure.
259 @return None
260 """
261 psf = exposure.getPsf()
262 assert sources.getSchema() == self.schema
263 self.deblend(exposure, sources, psf)
265 def _getPsfFwhm(self, psf, bbox):
266 # It should be easier to get a PSF's fwhm;
267 # https://dev.lsstcorp.org/trac/ticket/3030
268 return psf.computeShape().getDeterminantRadius() * 2.35
270 @pipeBase.timeMethod
271 def deblend(self, exposure, srcs, psf):
272 """!
273 Deblend.
275 @param[in] exposure Exposure to process
276 @param[in,out] srcs SourceCatalog containing sources detected on this exposure.
277 @param[in] psf PSF
279 @return None
280 """
281 self.log.info("Deblending %d sources" % len(srcs))
283 from lsst.meas.deblender.baseline import deblend
285 # find the median stdev in the image...
286 mi = exposure.getMaskedImage()
287 statsCtrl = afwMath.StatisticsControl()
288 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
289 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
290 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
291 self.log.trace('sigma1: %g', sigma1)
293 n0 = len(srcs)
294 nparents = 0
295 for i, src in enumerate(srcs):
296 # t0 = time.clock()
298 fp = src.getFootprint()
299 pks = fp.getPeaks()
301 # Since we use the first peak for the parent object, we should propagate its flags
302 # to the parent source.
303 src.assign(pks[0], self.peakSchemaMapper)
305 if len(pks) < 2:
306 continue
308 if self.isLargeFootprint(fp):
309 src.set(self.tooBigKey, True)
310 self.skipParent(src, mi.getMask())
311 self.log.warn('Parent %i: skipping large footprint (area: %i)',
312 int(src.getId()), int(fp.getArea()))
313 continue
314 if self.isMasked(fp, exposure.getMaskedImage().getMask()):
315 src.set(self.maskedKey, True)
316 self.skipParent(src, mi.getMask())
317 self.log.warn('Parent %i: skipping masked footprint (area: %i)',
318 int(src.getId()), int(fp.getArea()))
319 continue
321 nparents += 1
322 bb = fp.getBBox()
323 psf_fwhm = self._getPsfFwhm(psf, bb)
325 self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks))
327 self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
328 npre = len(srcs)
330 # This should really be set in deblend, but deblend doesn't have access to the src
331 src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
333 try:
334 res = deblend(
335 fp, mi, psf, psf_fwhm, sigma1=sigma1,
336 psfChisqCut1=self.config.psfChisq1,
337 psfChisqCut2=self.config.psfChisq2,
338 psfChisqCut2b=self.config.psfChisq2b,
339 maxNumberOfPeaks=self.config.maxNumberOfPeaks,
340 strayFluxToPointSources=self.config.strayFluxToPointSources,
341 assignStrayFlux=self.config.assignStrayFlux,
342 strayFluxAssignment=self.config.strayFluxRule,
343 rampFluxAtEdge=(self.config.edgeHandling == 'ramp'),
344 patchEdges=(self.config.edgeHandling == 'noclip'),
345 tinyFootprintSize=self.config.tinyFootprintSize,
346 clipStrayFluxFraction=self.config.clipStrayFluxFraction,
347 weightTemplates=self.config.weightTemplates,
348 removeDegenerateTemplates=self.config.removeDegenerateTemplates,
349 maxTempDotProd=self.config.maxTempDotProd,
350 medianSmoothTemplate=self.config.medianSmoothTemplate
351 )
352 if self.config.catchFailures:
353 src.set(self.deblendFailedKey, False)
354 except Exception as e:
355 if self.config.catchFailures:
356 self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e))
357 src.set(self.deblendFailedKey, True)
358 import traceback
359 traceback.print_exc()
360 continue
361 else:
362 raise
364 kids = []
365 nchild = 0
366 for j, peak in enumerate(res.deblendedParents[0].peaks):
367 heavy = peak.getFluxPortion()
368 if heavy is None or peak.skip:
369 src.set(self.deblendSkippedKey, True)
370 if not self.config.propagateAllPeaks:
371 # Don't care
372 continue
373 # We need to preserve the peak: make sure we have enough info to create a minimal
374 # child src
375 self.log.trace("Peak at (%i,%i) failed. Using minimal default info for child.",
376 pks[j].getIx(), pks[j].getIy())
377 if heavy is None:
378 # copy the full footprint and strip out extra peaks
379 foot = afwDet.Footprint(src.getFootprint())
380 peakList = foot.getPeaks()
381 peakList.clear()
382 peakList.append(peak.peak)
383 zeroMimg = afwImage.MaskedImageF(foot.getBBox())
384 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
385 if peak.deblendedAsPsf:
386 if peak.psfFitFlux is None:
387 peak.psfFitFlux = 0.0
388 if peak.psfFitCenter is None:
389 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
391 assert(len(heavy.getPeaks()) == 1)
393 src.set(self.deblendSkippedKey, False)
394 child = srcs.addNew()
395 nchild += 1
396 for key in self.toCopyFromParent:
397 child.set(key, src.get(key))
398 child.assign(heavy.getPeaks()[0], self.peakSchemaMapper)
399 child.setParent(src.getId())
400 child.setFootprint(heavy)
401 child.set(self.psfKey, peak.deblendedAsPsf)
402 child.set(self.hasStrayFluxKey, peak.strayFlux is not None)
403 if peak.deblendedAsPsf:
404 (cx, cy) = peak.psfFitCenter
405 child.set(self.psfCenterKey, geom.Point2D(cx, cy))
406 child.set(self.psfFluxKey, peak.psfFitFlux)
407 child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate)
408 child.set(self.deblendPatchedTemplateKey, peak.patched)
410 # Set the position of the peak from the parent footprint
411 # This will make it easier to match the same source across
412 # deblenders and across observations, where the peak
413 # position is unlikely to change unless enough time passes
414 # for a source to move on the sky.
415 child.set(self.peakCenter, geom.Point2I(pks[j].getIx(), pks[j].getIy()))
416 child.set(self.peakIdKey, pks[j].getId())
418 # The children have a single peak
419 child.set(self.nPeaksKey, 1)
420 # Set the number of peaks in the parent
421 child.set(self.parentNPeaksKey, len(pks))
423 kids.append(child)
425 # Child footprints may extend beyond the full extent of their parent's which
426 # results in a failure of the replace-by-noise code to reinstate these pixels
427 # to their original values. The following updates the parent footprint
428 # in-place to ensure it contains the full union of itself and all of its
429 # children's footprints.
430 spans = src.getFootprint().spans
431 for child in kids:
432 spans = spans.union(child.getFootprint().spans)
433 src.getFootprint().setSpans(spans)
435 src.set(self.nChildKey, nchild)
437 self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
438 # print('Deblending parent id', src.getId(), 'took', time.clock() - t0)
440 n1 = len(srcs)
441 self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
442 % (n0, nparents, n1-n0, n1))
444 def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1):
445 pass
447 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
448 pass
450 def isLargeFootprint(self, footprint):
451 """Returns whether a Footprint is large
453 'Large' is defined by thresholds on the area, size and axis ratio.
454 These may be disabled independently by configuring them to be non-positive.
456 This is principally intended to get rid of satellite streaks, which the
457 deblender or other downstream processing can have trouble dealing with
458 (e.g., multiple large HeavyFootprints can chew up memory).
459 """
460 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
461 return True
462 if self.config.maxFootprintSize > 0:
463 bbox = footprint.getBBox()
464 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
465 return True
466 if self.config.minFootprintAxisRatio > 0:
467 axes = afwEll.Axes(footprint.getShape())
468 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
469 return True
470 return False
472 def isMasked(self, footprint, mask):
473 """Returns whether the footprint violates the mask limits"""
474 size = float(footprint.getArea())
475 for maskName, limit in self.config.maskLimits.items():
476 maskVal = mask.getPlaneBitMask(maskName)
477 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels
478 if (size - unmaskedSpan.getArea())/size > limit:
479 return True
480 return False
482 def skipParent(self, source, mask):
483 """Indicate that the parent source is not being deblended
485 We set the appropriate flags and mask.
487 @param source The source to flag as skipped
488 @param mask The mask to update
489 """
490 fp = source.getFootprint()
491 source.set(self.deblendSkippedKey, True)
492 if self.config.notDeblendedMask:
493 mask.addMaskPlane(self.config.notDeblendedMask)
494 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
496 # Set the center of the parent
497 bbox = fp.getBBox()
498 centerX = int(bbox.getMinX()+bbox.getWidth()/2)
499 centerY = int(bbox.getMinY()+bbox.getHeight()/2)
500 source.set(self.peakCenter, geom.Point2I(centerX, centerY))
501 # There are no deblended children, so nChild = 0
502 source.set(self.nChildKey, 0)
503 # But we also want to know how many peaks that we would have
504 # deblended if the parent wasn't skipped.
505 source.set(self.nPeaksKey, len(fp.peaks))
506 # Top level parents are not a detected peak, so they have no peakId
507 source.set(self.peakIdKey, 0)
508 # Top level parents also have no parentNPeaks
509 source.set(self.parentNPeaksKey, 0)