22 __all__ = [
'SourceDeblendConfig',
'SourceDeblendTask']
29 import lsst.pipe.base
as pipeBase
36 from lsst.utils.timer
import timeMethod
43 edgeHandling = pexConfig.ChoiceField(
44 doc=
'What to do when a peak to be deblended is close to the edge of the image',
45 dtype=str, default=
'ramp',
47 'clip':
'Clip the template at the edge AND the mirror of the edge.',
48 'ramp':
'Ramp down flux at the image edge by the PSF',
49 'noclip':
'Ignore the edge when building the symmetric template.',
53 strayFluxToPointSources = pexConfig.ChoiceField(
54 doc=
'When the deblender should attribute stray flux to point sources',
55 dtype=str, default=
'necessary',
57 'necessary':
'When there is not an extended object in the footprint',
59 'never': (
'Never; stray flux will not be attributed to any deblended child '
60 'if the deblender thinks all peaks look like point sources'),
64 assignStrayFlux = pexConfig.Field(dtype=bool, default=
True,
65 doc=
'Assign stray flux (not claimed by any child in the deblender) '
66 'to deblend children.')
68 strayFluxRule = pexConfig.ChoiceField(
69 doc=
'How to split flux among peaks',
70 dtype=str, default=
'trim',
72 'r-to-peak':
'~ 1/(1+R^2) to the peak',
73 'r-to-footprint': (
'~ 1/(1+R^2) to the closest pixel in the footprint. '
74 'CAUTION: this can be computationally expensive on large footprints!'),
75 'nearest-footprint': (
'Assign 100% to the nearest footprint (using L-1 norm aka '
76 'Manhattan distance)'),
77 'trim': (
'Shrink the parent footprint to pixels that are not assigned to children')
81 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
82 doc=(
'When splitting stray flux, clip fractions below '
83 'this value to zero.'))
84 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
85 doc=(
'Chi-squared per DOF cut for deciding a source is '
86 'a PSF during deblending (un-shifted PSF model)'))
87 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
88 doc=(
'Chi-squared per DOF cut for deciding a source is '
89 'PSF during deblending (shifted PSF model)'))
90 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=
False,
91 doc=(
'Chi-squared per DOF cut for deciding a source is '
92 'a PSF during deblending (shifted PSF model #2)'))
93 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0,
94 doc=(
"Only deblend the brightest maxNumberOfPeaks peaks in the parent"
95 " (<= 0: unlimited)"))
96 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000,
97 doc=(
"Maximum area for footprints before they are ignored as large; "
98 "non-positive means no threshold applied"))
99 maxFootprintSize = pexConfig.Field(dtype=int, default=0,
100 doc=(
"Maximum linear dimension for footprints before they are ignored "
101 "as large; non-positive means no threshold applied"))
102 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0,
103 doc=(
"Minimum axis ratio for footprints before they are ignored "
104 "as large; non-positive means no threshold applied"))
105 notDeblendedMask = pexConfig.Field(dtype=str, default=
"NOT_DEBLENDED", optional=
True,
106 doc=
"Mask name for footprints not deblended, or None")
108 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=
True,
109 doc=(
'Footprints smaller in width or height than this value '
110 'will be ignored; minimum of 2 due to PSF gradient '
113 propagateAllPeaks = pexConfig.Field(dtype=bool, default=
False,
114 doc=(
'Guarantee that all peaks produce a child source.'))
115 catchFailures = pexConfig.Field(
116 dtype=bool, default=
False,
117 doc=(
"If True, catch exceptions thrown by the deblender, log them, "
118 "and set a flag on the parent, instead of letting them propagate up"))
119 maskPlanes = pexConfig.ListField(dtype=str, default=[
"SAT",
"INTRP",
"NO_DATA"],
120 doc=
"Mask planes to ignore when performing statistics")
121 maskLimits = pexConfig.DictField(
125 doc=(
"Mask planes with the corresponding limit on the fraction of masked pixels. "
126 "Sources violating this limit will not be deblended."),
128 weightTemplates = pexConfig.Field(
129 dtype=bool, default=
False,
130 doc=(
"If true, a least-squares fit of the templates will be done to the "
131 "full image. The templates will be re-weighted based on this fit."))
132 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=
False,
133 doc=(
"Try to remove similar templates?"))
134 maxTempDotProd = pexConfig.Field(
135 dtype=float, default=0.5,
136 doc=(
"If the dot product between two templates is larger than this value, we consider them to be "
137 "describing the same object (i.e. they are degenerate). If one of the objects has been "
138 "labeled as a PSF it will be removed, otherwise the template with the lowest value will "
140 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=
True,
141 doc=
"Apply a smoothing filter to all of the template images")
150 useCiLimits = pexConfig.Field(
151 dtype=bool, default=
False,
152 doc=
"Limit the number of sources deblended for CI to prevent long build times")
153 ciDeblendChildRange = pexConfig.ListField(
154 dtype=int, default=[2, 10],
155 doc=
"Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
156 "If `useCiLimits==False` then this parameter is ignored.")
157 ciNumParentsToDeblend = pexConfig.Field(
158 dtype=int, default=10,
159 doc=
"Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
160 "within `ciDebledChildRange`. "
161 "If `useCiLimits==False` then this parameter is ignored.")
165 """Split blended sources into individual sources.
167 This task has no return value; it only modifies the SourceCatalog in-place.
169 ConfigClass = SourceDeblendConfig
170 _DefaultName =
"sourceDeblend"
172 def __init__(self, schema, peakSchema=None, **kwargs):
173 """Create the task, adding necessary fields to the given schema.
177 schema : `lsst.afw.table.Schema`
178 Schema object for measurement fields; will be modified in-place.
179 peakSchema : `lsst.afw.table.peakSchema`
180 Schema of Footprint Peaks that will be passed to the deblender.
181 Any fields beyond the PeakTable minimal schema will be transferred
182 to the main source Schema. If None, no fields will be transferred
185 Additional keyword arguments passed to ~lsst.pipe.base.task
187 pipeBase.Task.__init__(self, **kwargs)
190 if item.field.getName().startswith(
"merge_footprint")]
191 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
192 if peakSchema
is None:
197 self.
peakSchemaMapperpeakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
198 for item
in peakSchema:
199 if item.key
not in peakMinimalSchema:
205 schema.addField(item.field)
206 assert schema == self.
peakSchemaMapperpeakSchemaMapper.getOutputSchema(),
"Logic bug mapping schemas"
210 self.
nChildKeynChildKey = schema.addField(
'deblend_nChild', type=np.int32,
211 doc=
'Number of children this object has (defaults to 0)')
212 self.
psfKeypsfKey = schema.addField(
'deblend_deblendedAsPsf', type=
'Flag',
213 doc=
'Deblender thought this source looked like a PSF')
214 self.
psfCenterKeypsfCenterKey = afwTable.Point2DKey.addFields(schema,
'deblend_psfCenter',
215 'If deblended-as-psf, the PSF centroid',
"pixel")
216 self.
psfFluxKeypsfFluxKey = schema.addField(
'deblend_psf_instFlux', type=
'D',
217 doc=
'If deblended-as-psf, the instrumental PSF flux', units=
'count')
218 self.
tooManyPeaksKeytooManyPeaksKey = schema.addField(
'deblend_tooManyPeaks', type=
'Flag',
219 doc=
'Source had too many peaks; '
220 'only the brightest were included')
221 self.
tooBigKeytooBigKey = schema.addField(
'deblend_parentTooBig', type=
'Flag',
222 doc=
'Parent footprint covered too many pixels')
223 self.
maskedKeymaskedKey = schema.addField(
'deblend_masked', type=
'Flag',
224 doc=
'Parent footprint was predominantly masked')
226 if self.config.catchFailures:
228 doc=
"Deblending failed on source")
231 doc=
"Deblender skipped this source")
234 'deblend_rampedTemplate', type=
'Flag',
235 doc=(
'This source was near an image edge and the deblender used '
236 '"ramp" edge-handling.'))
239 'deblend_patchedTemplate', type=
'Flag',
240 doc=(
'This source was near an image edge and the deblender used '
241 '"patched" edge-handling.'))
244 'deblend_hasStrayFlux', type=
'Flag',
245 doc=(
'This source was assigned some stray flux'))
247 self.log.trace(
'Added keys to schema: %s',
", ".join(str(x)
for x
in (
250 self.
peakCenterpeakCenter = afwTable.Point2IKey.addFields(schema, name=
"deblend_peak_center",
251 doc=
"Center used to apply constraints in scarlet",
253 self.
peakIdKeypeakIdKey = schema.addField(
"deblend_peakId", type=np.int32,
254 doc=
"ID of the peak in the parent footprint. "
255 "This is not unique, but the combination of 'parent'"
256 "and 'peakId' should be for all child sources. "
257 "Top level blends with no parents have 'peakId=0'")
258 self.
nPeaksKeynPeaksKey = schema.addField(
"deblend_nPeaks", type=np.int32,
259 doc=
"Number of initial peaks in the blend. "
260 "This includes peaks that may have been culled "
261 "during deblending or failed to deblend")
262 self.
parentNPeaksKeyparentNPeaksKey = schema.addField(
"deblend_parentNPeaks", type=np.int32,
263 doc=
"Same as deblend_n_peaks, but the number of peaks "
264 "in the parent footprint")
267 def run(self, exposure, sources):
268 """Get the PSF from the provided exposure and then run deblend.
272 exposure : `lsst.afw.image.Exposure`
273 Exposure to be processed
274 sources : `lsst.afw.table.SourceCatalog`
275 SourceCatalog containing sources detected on this exposure.
277 psf = exposure.getPsf()
278 assert sources.getSchema() == self.
schemaschema
279 self.
deblenddeblend(exposure, sources, psf)
281 def _getPsfFwhm(self, psf, bbox):
284 return psf.computeShape().getDeterminantRadius() * 2.35
292 exposure : `lsst.afw.image.Exposure`
293 Exposure to be processed
294 srcs : `lsst.afw.table.SourceCatalog`
295 SourceCatalog containing sources detected on this exposure
296 psf : `lsst.afw.detection.Psf`
297 Point source function
304 if self.config.useCiLimits:
305 self.log.info(f
"Using CI catalog limits, "
306 f
"the original number of sources to deblend was {len(srcs)}.")
309 minChildren, maxChildren = self.config.ciDeblendChildRange
310 nPeaks = np.array([len(src.getFootprint().peaks)
for src
in srcs])
311 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
312 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
313 raise ValueError(
"Fewer than ciNumParentsToDeblend children were contained in the range "
314 "indicated by ciDeblendChildRange. Adjust this range to include more "
318 parents = nPeaks == 1
319 children = np.zeros((len(srcs),), dtype=bool)
320 children[childrenInRange[:self.config.ciNumParentsToDeblend]] =
True
321 srcs = srcs[parents | children]
324 idFactory = srcs.getIdFactory()
325 maxId = np.max(srcs[
"id"])
326 idFactory.notify(maxId)
328 self.log.info(
"Deblending %d sources" % len(srcs))
333 mi = exposure.getMaskedImage()
334 statsCtrl = afwMath.StatisticsControl()
335 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
336 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
337 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
338 self.log.trace(
'sigma1: %g', sigma1)
342 for i, src
in enumerate(srcs):
345 fp = src.getFootprint()
358 self.log.warn(
'Parent %i: skipping large footprint (area: %i)',
359 int(src.getId()), int(fp.getArea()))
361 if self.
isMaskedisMasked(fp, exposure.getMaskedImage().getMask()):
364 self.log.warn(
'Parent %i: skipping masked footprint (area: %i)',
365 int(src.getId()), int(fp.getArea()))
372 self.log.trace(
'Parent %i: deblending %i peaks', int(src.getId()), len(pks))
378 src.set(self.
tooManyPeaksKeytooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
382 fp, mi, psf, psf_fwhm, sigma1=sigma1,
383 psfChisqCut1=self.config.psfChisq1,
384 psfChisqCut2=self.config.psfChisq2,
385 psfChisqCut2b=self.config.psfChisq2b,
386 maxNumberOfPeaks=self.config.maxNumberOfPeaks,
387 strayFluxToPointSources=self.config.strayFluxToPointSources,
388 assignStrayFlux=self.config.assignStrayFlux,
389 strayFluxAssignment=self.config.strayFluxRule,
390 rampFluxAtEdge=(self.config.edgeHandling ==
'ramp'),
391 patchEdges=(self.config.edgeHandling ==
'noclip'),
392 tinyFootprintSize=self.config.tinyFootprintSize,
393 clipStrayFluxFraction=self.config.clipStrayFluxFraction,
394 weightTemplates=self.config.weightTemplates,
395 removeDegenerateTemplates=self.config.removeDegenerateTemplates,
396 maxTempDotProd=self.config.maxTempDotProd,
397 medianSmoothTemplate=self.config.medianSmoothTemplate
399 if self.config.catchFailures:
401 except Exception
as e:
402 if self.config.catchFailures:
403 self.log.warn(
"Unable to deblend source %d: %s" % (src.getId(), e))
406 traceback.print_exc()
413 for j, peak
in enumerate(res.deblendedParents[0].peaks):
414 heavy = peak.getFluxPortion()
415 if heavy
is None or peak.skip:
417 if not self.config.propagateAllPeaks:
422 self.log.trace(
"Peak at (%i,%i) failed. Using minimal default info for child.",
423 pks[j].getIx(), pks[j].getIy())
426 foot = afwDet.Footprint(src.getFootprint())
427 peakList = foot.getPeaks()
429 peakList.append(peak.peak)
430 zeroMimg = afwImage.MaskedImageF(foot.getBBox())
431 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
432 if peak.deblendedAsPsf:
433 if peak.psfFitFlux
is None:
434 peak.psfFitFlux = 0.0
435 if peak.psfFitCenter
is None:
436 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
438 assert(len(heavy.getPeaks()) == 1)
441 child = srcs.addNew()
444 child.set(key, src.get(key))
446 child.setParent(src.getId())
447 child.setFootprint(heavy)
448 child.set(self.
psfKeypsfKey, peak.deblendedAsPsf)
449 child.set(self.
hasStrayFluxKeyhasStrayFluxKey, peak.strayFlux
is not None)
450 if peak.deblendedAsPsf:
451 (cx, cy) = peak.psfFitCenter
453 child.set(self.
psfFluxKeypsfFluxKey, peak.psfFitFlux)
463 child.set(self.
peakIdKeypeakIdKey, pks[j].getId())
477 spans = src.getFootprint().spans
479 spans = spans.union(child.getFootprint().spans)
480 src.getFootprint().setSpans(spans)
484 self.
postSingleDeblendHookpostSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
488 self.log.info(
'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
489 % (n0, nparents, n1-n0, n1))
494 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
498 """Returns whether a Footprint is large
500 'Large' is defined by thresholds on the area, size and axis ratio.
501 These may be disabled independently by configuring them to be non-positive.
503 This is principally intended to get rid of satellite streaks, which the
504 deblender or other downstream processing can have trouble dealing with
505 (e.g., multiple large HeavyFootprints can chew up memory).
507 if self.config.maxFootprintArea > 0
and footprint.getArea() > self.config.maxFootprintArea:
509 if self.config.maxFootprintSize > 0:
510 bbox = footprint.getBBox()
511 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
513 if self.config.minFootprintAxisRatio > 0:
514 axes = afwEll.Axes(footprint.getShape())
515 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
520 """Returns whether the footprint violates the mask limits
522 size = float(footprint.getArea())
523 for maskName, limit
in self.config.maskLimits.items():
524 maskVal = mask.getPlaneBitMask(maskName)
525 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal)
526 if (size - unmaskedSpan.getArea())/size > limit:
531 """Indicate that the parent source is not being deblended
533 We set the appropriate flags and mask.
537 source : `lsst.afw.table.SourceRecord`
538 The source to flag as skipped
539 mask : `lsst.afw.image.Mask`
542 fp = source.getFootprint()
544 if self.config.notDeblendedMask:
545 mask.addMaskPlane(self.config.notDeblendedMask)
546 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
550 centerX = int(bbox.getMinX()+bbox.getWidth()/2)
551 centerY = int(bbox.getMinY()+bbox.getHeight()/2)
557 source.set(self.
nPeaksKeynPeaksKey, len(fp.peaks))
static Log getLogger(std::string const &loggername)
def isLargeFootprint(self, footprint)
def deblend(self, exposure, srcs, psf)
def addSchemaKeys(self, schema)
def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
def isMasked(self, footprint, mask)
def skipParent(self, source, mask)
def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
def __init__(self, schema, peakSchema=None, **kwargs)
def _getPsfFwhm(self, psf, bbox)
deblendPatchedTemplateKey
def run(self, exposure, sources)