27 import lsst.pipe.base
as pipeBase
37 __all__ =
'SourceDeblendConfig',
'SourceDeblendTask'
42 edgeHandling = pexConfig.ChoiceField(
43 doc=
'What to do when a peak to be deblended is close to the edge of the image',
44 dtype=str, default=
'ramp',
46 'clip':
'Clip the template at the edge AND the mirror of the edge.',
47 'ramp':
'Ramp down flux at the image edge by the PSF',
48 'noclip':
'Ignore the edge when building the symmetric template.',
52 strayFluxToPointSources = pexConfig.ChoiceField(
53 doc=
'When the deblender should attribute stray flux to point sources',
54 dtype=str, default=
'necessary',
56 'necessary':
'When there is not an extended object in the footprint',
58 'never': (
'Never; stray flux will not be attributed to any deblended child '
59 'if the deblender thinks all peaks look like point sources'),
63 assignStrayFlux = pexConfig.Field(dtype=bool, default=
True,
64 doc=
'Assign stray flux (not claimed by any child in the deblender) '
65 'to deblend children.')
67 strayFluxRule = pexConfig.ChoiceField(
68 doc=
'How to split flux among peaks',
69 dtype=str, default=
'trim',
71 'r-to-peak':
'~ 1/(1+R^2) to the peak',
72 'r-to-footprint': (
'~ 1/(1+R^2) to the closest pixel in the footprint. '
73 'CAUTION: this can be computationally expensive on large footprints!'),
74 'nearest-footprint': (
'Assign 100% to the nearest footprint (using L-1 norm aka '
75 'Manhattan distance)'),
76 'trim': (
'Shrink the parent footprint to pixels that are not assigned to children')
80 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
81 doc=(
'When splitting stray flux, clip fractions below '
82 'this value to zero.'))
83 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
84 doc=(
'Chi-squared per DOF cut for deciding a source is '
85 'a PSF during deblending (un-shifted PSF model)'))
86 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
87 doc=(
'Chi-squared per DOF cut for deciding a source is '
88 'PSF during deblending (shifted PSF model)'))
89 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=
False,
90 doc=(
'Chi-squared per DOF cut for deciding a source is '
91 'a PSF during deblending (shifted PSF model #2)'))
92 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0,
93 doc=(
"Only deblend the brightest maxNumberOfPeaks peaks in the parent"
94 " (<= 0: unlimited)"))
95 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000,
96 doc=(
"Maximum area for footprints before they are ignored as large; "
97 "non-positive means no threshold applied"))
98 maxFootprintSize = pexConfig.Field(dtype=int, default=0,
99 doc=(
"Maximum linear dimension for footprints before they are ignored "
100 "as large; non-positive means no threshold applied"))
101 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0,
102 doc=(
"Minimum axis ratio for footprints before they are ignored "
103 "as large; non-positive means no threshold applied"))
104 notDeblendedMask = pexConfig.Field(dtype=str, default=
"NOT_DEBLENDED", optional=
True,
105 doc=
"Mask name for footprints not deblended, or None")
107 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=
True,
108 doc=(
'Footprints smaller in width or height than this value '
109 'will be ignored; minimum of 2 due to PSF gradient '
112 propagateAllPeaks = pexConfig.Field(dtype=bool, default=
False,
113 doc=(
'Guarantee that all peaks produce a child source.'))
114 catchFailures = pexConfig.Field(
115 dtype=bool, default=
False,
116 doc=(
"If True, catch exceptions thrown by the deblender, log them, "
117 "and set a flag on the parent, instead of letting them propagate up"))
118 maskPlanes = pexConfig.ListField(dtype=str, default=[
"SAT",
"INTRP",
"NO_DATA"],
119 doc=
"Mask planes to ignore when performing statistics")
120 maskLimits = pexConfig.DictField(
124 doc=(
"Mask planes with the corresponding limit on the fraction of masked pixels. "
125 "Sources violating this limit will not be deblended."),
127 weightTemplates = pexConfig.Field(
128 dtype=bool, default=
False,
129 doc=(
"If true, a least-squares fit of the templates will be done to the "
130 "full image. The templates will be re-weighted based on this fit."))
131 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=
False,
132 doc=(
"Try to remove similar templates?"))
133 maxTempDotProd = pexConfig.Field(
134 dtype=float, default=0.5,
135 doc=(
"If the dot product between two templates is larger than this value, we consider them to be "
136 "describing the same object (i.e. they are degenerate). If one of the objects has been "
137 "labeled as a PSF it will be removed, otherwise the template with the lowest value will "
139 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=
True,
140 doc=
"Apply a smoothing filter to all of the template images")
149 useCiLimits = pexConfig.Field(
150 dtype=bool, default=
False,
151 doc=
"Limit the number of sources deblended for CI to prevent long build times")
152 ciDeblendChildRange = pexConfig.ListField(
153 dtype=int, default=[2, 10],
154 doc=
"Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
155 "If `useCiLimits==False` then this parameter is ignored.")
156 ciNumParentsToDeblend = pexConfig.Field(
157 dtype=int, default=10,
158 doc=
"Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
159 "within `ciDebledChildRange`. "
160 "If `useCiLimits==False` then this parameter is ignored.")
172 \anchor SourceDeblendTask_
174 \brief Split blended sources into individual sources.
176 This task has no return value; it only modifies the SourceCatalog in-place.
178 ConfigClass = SourceDeblendConfig
179 _DefaultName =
"sourceDeblend"
181 def __init__(self, schema, peakSchema=None, **kwargs):
183 Create the task, adding necessary fields to the given schema.
185 @param[in,out] schema Schema object for measurement fields; will be modified in-place.
186 @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender.
187 Any fields beyond the PeakTable minimal schema will be transferred
188 to the main source Schema. If None, no fields will be transferred
190 @param[in] **kwargs Passed to Task.__init__.
192 pipeBase.Task.__init__(self, **kwargs)
195 if item.field.getName().startswith(
"merge_footprint")]
196 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
197 if peakSchema
is None:
202 self.
peakSchemaMapperpeakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
203 for item
in peakSchema:
204 if item.key
not in peakMinimalSchema:
210 schema.addField(item.field)
211 assert schema == self.
peakSchemaMapperpeakSchemaMapper.getOutputSchema(),
"Logic bug mapping schemas"
215 self.
nChildKeynChildKey = schema.addField(
'deblend_nChild', type=np.int32,
216 doc=
'Number of children this object has (defaults to 0)')
217 self.
psfKeypsfKey = schema.addField(
'deblend_deblendedAsPsf', type=
'Flag',
218 doc=
'Deblender thought this source looked like a PSF')
219 self.
psfCenterKeypsfCenterKey = afwTable.Point2DKey.addFields(schema,
'deblend_psfCenter',
220 'If deblended-as-psf, the PSF centroid',
"pixel")
221 self.
psfFluxKeypsfFluxKey = schema.addField(
'deblend_psf_instFlux', type=
'D',
222 doc=
'If deblended-as-psf, the instrumental PSF flux', units=
'count')
223 self.
tooManyPeaksKeytooManyPeaksKey = schema.addField(
'deblend_tooManyPeaks', type=
'Flag',
224 doc=
'Source had too many peaks; '
225 'only the brightest were included')
226 self.
tooBigKeytooBigKey = schema.addField(
'deblend_parentTooBig', type=
'Flag',
227 doc=
'Parent footprint covered too many pixels')
228 self.
maskedKeymaskedKey = schema.addField(
'deblend_masked', type=
'Flag',
229 doc=
'Parent footprint was predominantly masked')
231 if self.config.catchFailures:
233 doc=
"Deblending failed on source")
236 doc=
"Deblender skipped this source")
239 'deblend_rampedTemplate', type=
'Flag',
240 doc=(
'This source was near an image edge and the deblender used '
241 '"ramp" edge-handling.'))
244 'deblend_patchedTemplate', type=
'Flag',
245 doc=(
'This source was near an image edge and the deblender used '
246 '"patched" edge-handling.'))
249 'deblend_hasStrayFlux', type=
'Flag',
250 doc=(
'This source was assigned some stray flux'))
252 self.log.trace(
'Added keys to schema: %s',
", ".join(str(x)
for x
in (
255 self.
peakCenterpeakCenter = afwTable.Point2IKey.addFields(schema, name=
"deblend_peak_center",
256 doc=
"Center used to apply constraints in scarlet",
258 self.
peakIdKeypeakIdKey = schema.addField(
"deblend_peakId", type=np.int32,
259 doc=
"ID of the peak in the parent footprint. "
260 "This is not unique, but the combination of 'parent'"
261 "and 'peakId' should be for all child sources. "
262 "Top level blends with no parents have 'peakId=0'")
263 self.
nPeaksKeynPeaksKey = schema.addField(
"deblend_nPeaks", type=np.int32,
264 doc=
"Number of initial peaks in the blend. "
265 "This includes peaks that may have been culled "
266 "during deblending or failed to deblend")
267 self.
parentNPeaksKeyparentNPeaksKey = schema.addField(
"deblend_parentNPeaks", type=np.int32,
268 doc=
"Same as deblend_n_peaks, but the number of peaks "
269 "in the parent footprint")
272 def run(self, exposure, sources):
274 Get the psf from the provided exposure and then run deblend().
276 @param[in] exposure Exposure to process
277 @param[in,out] sources SourceCatalog containing sources detected on this exposure.
281 psf = exposure.getPsf()
282 assert sources.getSchema() == self.
schemaschema
283 self.
deblenddeblend(exposure, sources, psf)
285 def _getPsfFwhm(self, psf, bbox):
288 return psf.computeShape().getDeterminantRadius() * 2.35
295 @param[in] exposure Exposure to process
296 @param[in,out] srcs SourceCatalog containing sources detected on this exposure.
302 if self.config.useCiLimits:
303 self.log.info(f
"Using CI catalog limits, "
304 f
"the original number of sources to deblend was {len(srcs)}.")
307 minChildren, maxChildren = self.config.ciDeblendChildRange
308 nPeaks = np.array([len(src.getFootprint().peaks)
for src
in srcs])
309 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
310 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
311 raise ValueError(
"Fewer than ciNumParentsToDeblend children were contained in the range "
312 "indicated by ciDeblendChildRange. Adjust this range to include more "
316 parents = nPeaks == 1
317 children = np.zeros((len(srcs),), dtype=bool)
318 children[childrenInRange[:self.config.ciNumParentsToDeblend]] =
True
319 srcs = srcs[parents | children]
322 idFactory = srcs.getIdFactory()
323 maxId = np.max(srcs[
"id"])
324 idFactory.notify(maxId)
326 self.log.info(
"Deblending %d sources" % len(srcs))
331 mi = exposure.getMaskedImage()
332 statsCtrl = afwMath.StatisticsControl()
333 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
334 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
335 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
336 self.log.trace(
'sigma1: %g', sigma1)
340 for i, src
in enumerate(srcs):
343 fp = src.getFootprint()
356 self.log.warn(
'Parent %i: skipping large footprint (area: %i)',
357 int(src.getId()), int(fp.getArea()))
359 if self.
isMaskedisMasked(fp, exposure.getMaskedImage().getMask()):
362 self.log.warn(
'Parent %i: skipping masked footprint (area: %i)',
363 int(src.getId()), int(fp.getArea()))
370 self.log.trace(
'Parent %i: deblending %i peaks', int(src.getId()), len(pks))
376 src.set(self.
tooManyPeaksKeytooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
380 fp, mi, psf, psf_fwhm, sigma1=sigma1,
381 psfChisqCut1=self.config.psfChisq1,
382 psfChisqCut2=self.config.psfChisq2,
383 psfChisqCut2b=self.config.psfChisq2b,
384 maxNumberOfPeaks=self.config.maxNumberOfPeaks,
385 strayFluxToPointSources=self.config.strayFluxToPointSources,
386 assignStrayFlux=self.config.assignStrayFlux,
387 strayFluxAssignment=self.config.strayFluxRule,
388 rampFluxAtEdge=(self.config.edgeHandling ==
'ramp'),
389 patchEdges=(self.config.edgeHandling ==
'noclip'),
390 tinyFootprintSize=self.config.tinyFootprintSize,
391 clipStrayFluxFraction=self.config.clipStrayFluxFraction,
392 weightTemplates=self.config.weightTemplates,
393 removeDegenerateTemplates=self.config.removeDegenerateTemplates,
394 maxTempDotProd=self.config.maxTempDotProd,
395 medianSmoothTemplate=self.config.medianSmoothTemplate
397 if self.config.catchFailures:
399 except Exception
as e:
400 if self.config.catchFailures:
401 self.log.warn(
"Unable to deblend source %d: %s" % (src.getId(), e))
404 traceback.print_exc()
411 for j, peak
in enumerate(res.deblendedParents[0].peaks):
412 heavy = peak.getFluxPortion()
413 if heavy
is None or peak.skip:
415 if not self.config.propagateAllPeaks:
420 self.log.trace(
"Peak at (%i,%i) failed. Using minimal default info for child.",
421 pks[j].getIx(), pks[j].getIy())
424 foot = afwDet.Footprint(src.getFootprint())
425 peakList = foot.getPeaks()
427 peakList.append(peak.peak)
428 zeroMimg = afwImage.MaskedImageF(foot.getBBox())
429 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
430 if peak.deblendedAsPsf:
431 if peak.psfFitFlux
is None:
432 peak.psfFitFlux = 0.0
433 if peak.psfFitCenter
is None:
434 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
436 assert(len(heavy.getPeaks()) == 1)
439 child = srcs.addNew()
442 child.set(key, src.get(key))
444 child.setParent(src.getId())
445 child.setFootprint(heavy)
446 child.set(self.
psfKeypsfKey, peak.deblendedAsPsf)
447 child.set(self.
hasStrayFluxKeyhasStrayFluxKey, peak.strayFlux
is not None)
448 if peak.deblendedAsPsf:
449 (cx, cy) = peak.psfFitCenter
451 child.set(self.
psfFluxKeypsfFluxKey, peak.psfFitFlux)
461 child.set(self.
peakIdKeypeakIdKey, pks[j].getId())
475 spans = src.getFootprint().spans
477 spans = spans.union(child.getFootprint().spans)
478 src.getFootprint().setSpans(spans)
482 self.
postSingleDeblendHookpostSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
486 self.log.info(
'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
487 % (n0, nparents, n1-n0, n1))
492 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
496 """Returns whether a Footprint is large
498 'Large' is defined by thresholds on the area, size and axis ratio.
499 These may be disabled independently by configuring them to be non-positive.
501 This is principally intended to get rid of satellite streaks, which the
502 deblender or other downstream processing can have trouble dealing with
503 (e.g., multiple large HeavyFootprints can chew up memory).
505 if self.config.maxFootprintArea > 0
and footprint.getArea() > self.config.maxFootprintArea:
507 if self.config.maxFootprintSize > 0:
508 bbox = footprint.getBBox()
509 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
511 if self.config.minFootprintAxisRatio > 0:
512 axes = afwEll.Axes(footprint.getShape())
513 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
518 """Returns whether the footprint violates the mask limits"""
519 size = float(footprint.getArea())
520 for maskName, limit
in self.config.maskLimits.items():
521 maskVal = mask.getPlaneBitMask(maskName)
522 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal)
523 if (size - unmaskedSpan.getArea())/size > limit:
528 """Indicate that the parent source is not being deblended
530 We set the appropriate flags and mask.
532 @param source The source to flag as skipped
533 @param mask The mask to update
535 fp = source.getFootprint()
537 if self.config.notDeblendedMask:
538 mask.addMaskPlane(self.config.notDeblendedMask)
539 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
543 centerX = int(bbox.getMinX()+bbox.getWidth()/2)
544 centerY = int(bbox.getMinY()+bbox.getHeight()/2)
550 source.set(self.
nPeaksKeynPeaksKey, len(fp.peaks))
static Log getLogger(std::string const &loggername)
def isLargeFootprint(self, footprint)
def deblend(self, exposure, srcs, psf)
Deblend.
def addSchemaKeys(self, schema)
def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
def isMasked(self, footprint, mask)
def skipParent(self, source, mask)
def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
def __init__(self, schema, peakSchema=None, **kwargs)
Create the task, adding necessary fields to the given schema.
def _getPsfFwhm(self, psf, bbox)
deblendPatchedTemplateKey
def run(self, exposure, sources)
Get the psf from the provided exposure and then run deblend().