40 __all__ =
'SourceDeblendConfig',
'SourceDeblendTask',
'MultibandDeblendConfig',
'MultibandDeblendTask' 45 edgeHandling = pexConfig.ChoiceField(
46 doc=
'What to do when a peak to be deblended is close to the edge of the image',
47 dtype=str, default=
'ramp',
49 'clip':
'Clip the template at the edge AND the mirror of the edge.',
50 'ramp':
'Ramp down flux at the image edge by the PSF',
51 'noclip':
'Ignore the edge when building the symmetric template.',
55 strayFluxToPointSources = pexConfig.ChoiceField(
56 doc=
'When the deblender should attribute stray flux to point sources',
57 dtype=str, default=
'necessary',
59 'necessary':
'When there is not an extended object in the footprint',
61 'never': (
'Never; stray flux will not be attributed to any deblended child ' 62 'if the deblender thinks all peaks look like point sources'),
66 assignStrayFlux = pexConfig.Field(dtype=bool, default=
True,
67 doc=
'Assign stray flux (not claimed by any child in the deblender) ' 68 'to deblend children.')
70 strayFluxRule = pexConfig.ChoiceField(
71 doc=
'How to split flux among peaks',
72 dtype=str, default=
'trim',
74 'r-to-peak':
'~ 1/(1+R^2) to the peak',
75 'r-to-footprint': (
'~ 1/(1+R^2) to the closest pixel in the footprint. ' 76 'CAUTION: this can be computationally expensive on large footprints!'),
77 'nearest-footprint': (
'Assign 100% to the nearest footprint (using L-1 norm aka ' 78 'Manhattan distance)'),
79 'trim': (
'Shrink the parent footprint to pixels that are not assigned to children')
83 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
84 doc=(
'When splitting stray flux, clip fractions below ' 85 'this value to zero.'))
86 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
87 doc=(
'Chi-squared per DOF cut for deciding a source is ' 88 'a PSF during deblending (un-shifted PSF model)'))
89 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
90 doc=(
'Chi-squared per DOF cut for deciding a source is ' 91 'PSF during deblending (shifted PSF model)'))
92 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=
False,
93 doc=(
'Chi-squared per DOF cut for deciding a source is ' 94 'a PSF during deblending (shifted PSF model #2)'))
95 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0,
96 doc=(
"Only deblend the brightest maxNumberOfPeaks peaks in the parent" 97 " (<= 0: unlimited)"))
98 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000,
99 doc=(
"Maximum area for footprints before they are ignored as large; " 100 "non-positive means no threshold applied"))
101 maxFootprintSize = pexConfig.Field(dtype=int, default=0,
102 doc=(
"Maximum linear dimension for footprints before they are ignored " 103 "as large; non-positive means no threshold applied"))
104 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0,
105 doc=(
"Minimum axis ratio for footprints before they are ignored " 106 "as large; non-positive means no threshold applied"))
107 notDeblendedMask = pexConfig.Field(dtype=str, default=
"NOT_DEBLENDED", optional=
True,
108 doc=
"Mask name for footprints not deblended, or None")
110 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=
True,
111 doc=(
'Footprints smaller in width or height than this value ' 112 'will be ignored; minimum of 2 due to PSF gradient ' 115 propagateAllPeaks = pexConfig.Field(dtype=bool, default=
False,
116 doc=(
'Guarantee that all peaks produce a child source.'))
117 catchFailures = pexConfig.Field(
118 dtype=bool, default=
False,
119 doc=(
"If True, catch exceptions thrown by the deblender, log them, " 120 "and set a flag on the parent, instead of letting them propagate up"))
121 maskPlanes = pexConfig.ListField(dtype=str, default=[
"SAT",
"INTRP",
"NO_DATA"],
122 doc=
"Mask planes to ignore when performing statistics")
123 maskLimits = pexConfig.DictField(
127 doc=(
"Mask planes with the corresponding limit on the fraction of masked pixels. " 128 "Sources violating this limit will not be deblended."),
130 weightTemplates = pexConfig.Field(
131 dtype=bool, default=
False,
132 doc=(
"If true, a least-squares fit of the templates will be done to the " 133 "full image. The templates will be re-weighted based on this fit."))
134 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=
False,
135 doc=(
"Try to remove similar templates?"))
136 maxTempDotProd = pexConfig.Field(
137 dtype=float, default=0.5,
138 doc=(
"If the dot product between two templates is larger than this value, we consider them to be " 139 "describing the same object (i.e. they are degenerate). If one of the objects has been " 140 "labeled as a PSF it will be removed, otherwise the template with the lowest value will " 142 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=
True,
143 doc=
"Apply a smoothing filter to all of the template images")
155 \anchor SourceDeblendTask_ 157 \brief Split blended sources into individual sources. 159 This task has no return value; it only modifies the SourceCatalog in-place. 161 ConfigClass = SourceDeblendConfig
162 _DefaultName =
"sourceDeblend" 164 def __init__(self, schema, peakSchema=None, **kwargs):
166 Create the task, adding necessary fields to the given schema. 168 @param[in,out] schema Schema object for measurement fields; will be modified in-place. 169 @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender. 170 Any fields beyond the PeakTable minimal schema will be transferred 171 to the main source Schema. If None, no fields will be transferred 173 @param[in] **kwargs Passed to Task.__init__. 175 pipeBase.Task.__init__(self, **kwargs)
178 if item.field.getName().startswith(
"merge_footprint")]
179 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
180 if peakSchema
is None:
186 for item
in peakSchema:
187 if item.key
not in peakMinimalSchema:
193 schema.addField(item.field)
194 assert schema == self.
peakSchemaMapper.getOutputSchema(),
"Logic bug mapping schemas" 198 self.
nChildKey = schema.addField(
'deblend_nChild', type=np.int32,
199 doc=
'Number of children this object has (defaults to 0)')
200 self.
psfKey = schema.addField(
'deblend_deblendedAsPsf', type=
'Flag',
201 doc=
'Deblender thought this source looked like a PSF')
202 self.
psfCenterKey = afwTable.Point2DKey.addFields(schema,
'deblend_psfCenter',
203 'If deblended-as-psf, the PSF centroid',
"pixel")
204 self.
psfFluxKey = schema.addField(
'deblend_psf_instFlux', type=
'D',
205 doc=
'If deblended-as-psf, the instrumental PSF flux', units=
'count')
207 doc=
'Source had too many peaks; ' 208 'only the brightest were included')
209 self.
tooBigKey = schema.addField(
'deblend_parentTooBig', type=
'Flag',
210 doc=
'Parent footprint covered too many pixels')
211 self.
maskedKey = schema.addField(
'deblend_masked', type=
'Flag',
212 doc=
'Parent footprint was predominantly masked')
214 if self.config.catchFailures:
216 doc=
"Deblending failed on source")
219 doc=
"Deblender skipped this source")
222 'deblend_rampedTemplate', type=
'Flag',
223 doc=(
'This source was near an image edge and the deblender used ' 224 '"ramp" edge-handling.'))
227 'deblend_patchedTemplate', type=
'Flag',
228 doc=(
'This source was near an image edge and the deblender used ' 229 '"patched" edge-handling.'))
232 'deblend_hasStrayFlux', type=
'Flag',
233 doc=(
'This source was assigned some stray flux'))
235 self.log.trace(
'Added keys to schema: %s',
", ".join(str(x)
for x
in (
240 def run(self, exposure, sources):
242 Get the psf from the provided exposure and then run deblend(). 244 @param[in] exposure Exposure to process 245 @param[in,out] sources SourceCatalog containing sources detected on this exposure. 249 psf = exposure.getPsf()
250 assert sources.getSchema() == self.
schema 251 self.
deblend(exposure, sources, psf)
253 def _getPsfFwhm(self, psf, bbox):
256 return psf.computeShape().getDeterminantRadius() * 2.35
263 @param[in] exposure Exposure to process 264 @param[in,out] srcs SourceCatalog containing sources detected on this exposure. 269 self.log.info(
"Deblending %d sources" % len(srcs))
274 mi = exposure.getMaskedImage()
275 statsCtrl = afwMath.StatisticsControl()
276 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
277 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
278 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
279 self.log.trace(
'sigma1: %g', sigma1)
283 for i, src
in enumerate(srcs):
286 fp = src.getFootprint()
299 self.log.warn(
'Parent %i: skipping large footprint (area: %i)',
300 int(src.getId()), int(fp.getArea()))
302 if self.
isMasked(fp, exposure.getMaskedImage().getMask()):
305 self.log.warn(
'Parent %i: skipping masked footprint (area: %i)',
306 int(src.getId()), int(fp.getArea()))
313 self.log.trace(
'Parent %i: deblending %i peaks', int(src.getId()), len(pks))
319 src.set(self.
tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
323 fp, mi, psf, psf_fwhm, sigma1=sigma1,
324 psfChisqCut1=self.config.psfChisq1,
325 psfChisqCut2=self.config.psfChisq2,
326 psfChisqCut2b=self.config.psfChisq2b,
327 maxNumberOfPeaks=self.config.maxNumberOfPeaks,
328 strayFluxToPointSources=self.config.strayFluxToPointSources,
329 assignStrayFlux=self.config.assignStrayFlux,
330 strayFluxAssignment=self.config.strayFluxRule,
331 rampFluxAtEdge=(self.config.edgeHandling ==
'ramp'),
332 patchEdges=(self.config.edgeHandling ==
'noclip'),
333 tinyFootprintSize=self.config.tinyFootprintSize,
334 clipStrayFluxFraction=self.config.clipStrayFluxFraction,
335 weightTemplates=self.config.weightTemplates,
336 removeDegenerateTemplates=self.config.removeDegenerateTemplates,
337 maxTempDotProd=self.config.maxTempDotProd,
338 medianSmoothTemplate=self.config.medianSmoothTemplate
340 if self.config.catchFailures:
342 except Exception
as e:
343 if self.config.catchFailures:
344 self.log.warn(
"Unable to deblend source %d: %s" % (src.getId(), e))
347 traceback.print_exc()
354 for j, peak
in enumerate(res.deblendedParents[0].peaks):
355 heavy = peak.getFluxPortion()
356 if heavy
is None or peak.skip:
358 if not self.config.propagateAllPeaks:
363 self.log.trace(
"Peak at (%i,%i) failed. Using minimal default info for child.",
364 pks[j].getIx(), pks[j].getIy())
367 foot = afwDet.Footprint(src.getFootprint())
368 peakList = foot.getPeaks()
370 peakList.append(peak.peak)
371 zeroMimg = afwImage.MaskedImageF(foot.getBBox())
372 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
373 if peak.deblendedAsPsf:
374 if peak.psfFitFlux
is None:
375 peak.psfFitFlux = 0.0
376 if peak.psfFitCenter
is None:
377 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
379 assert(len(heavy.getPeaks()) == 1)
382 child = srcs.addNew()
385 child.set(key, src.get(key))
387 child.setParent(src.getId())
388 child.setFootprint(heavy)
389 child.set(self.
psfKey, peak.deblendedAsPsf)
391 if peak.deblendedAsPsf:
392 (cx, cy) = peak.psfFitCenter
404 spans = src.getFootprint().spans
406 spans = spans.union(child.getFootprint().spans)
407 src.getFootprint().setSpans(spans)
415 self.log.info(
'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' 416 % (n0, nparents, n1-n0, n1))
421 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
425 """Returns whether a Footprint is large 427 'Large' is defined by thresholds on the area, size and axis ratio. 428 These may be disabled independently by configuring them to be non-positive. 430 This is principally intended to get rid of satellite streaks, which the 431 deblender or other downstream processing can have trouble dealing with 432 (e.g., multiple large HeavyFootprints can chew up memory). 434 if self.config.maxFootprintArea > 0
and footprint.getArea() > self.config.maxFootprintArea:
436 if self.config.maxFootprintSize > 0:
437 bbox = footprint.getBBox()
438 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
440 if self.config.minFootprintAxisRatio > 0:
441 axes = afwEll.Axes(footprint.getShape())
442 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
447 """Returns whether the footprint violates the mask limits""" 448 size = float(footprint.getArea())
449 for maskName, limit
in self.config.maskLimits.items():
450 maskVal = mask.getPlaneBitMask(maskName)
451 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal)
452 if (size - unmaskedSpan.getArea())/size > limit:
457 """Indicate that the parent source is not being deblended 459 We set the appropriate flags and mask. 461 @param source The source to flag as skipped 462 @param mask The mask to update 464 fp = source.getFootprint()
466 source.set(self.
nChildKey, len(fp.getPeaks()))
467 if self.config.notDeblendedMask:
468 mask.addMaskPlane(self.config.notDeblendedMask)
469 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
473 """MultibandDeblendConfig 475 Configuration for the multiband deblender. 476 The parameters are organized by the parameter types, which are 477 - Stopping Criteria: Used to determine if the fit has converged 478 - Position Fitting Criteria: Used to fit the positions of the peaks 479 - Constraints: Used to apply constraints to the peaks and their components 480 - Other: Parameters that don't fit into the above categories 483 maxIter = pexConfig.Field(dtype=int, default=200,
484 doc=(
"Maximum number of iterations to deblend a single parent"))
485 relativeError = pexConfig.Field(dtype=float, default=1e-3,
486 doc=(
"Relative error to use when determining stopping criteria"))
489 minTranslation = pexConfig.Field(dtype=float, default=1e-3,
490 doc=(
"A peak must be updated by at least 'minTranslation' (pixels)" 491 "or no update is performed." 492 "This field is ignored if fitPositions is False."))
493 refinementSkip = pexConfig.Field(dtype=int, default=10,
494 doc=(
"If fitPositions is True, the positions and box sizes are" 495 "updated on every 'refinementSkip' iterations."))
496 translationMethod = pexConfig.Field(dtype=str, default=
"default",
497 doc=(
"Method to use for fitting translations." 498 "Currently 'default' is the only available option," 499 "which performs a linear fit, but it is possible that we" 500 "will use galsim or some other method as a future option"))
501 edgeFluxThresh = pexConfig.Field(dtype=float, default=1.0,
502 doc=(
"Boxes are resized when the flux at an edge is " 503 "> edgeFluxThresh * background RMS"))
504 exactLipschitz = pexConfig.Field(dtype=bool, default=
False,
505 doc=(
"Calculate exact Lipschitz constant in every step" 506 "(True) or only calculate the approximate" 507 "Lipschitz constant with significant changes in A,S" 509 stepSlack = pexConfig.Field(dtype=float, default=0.2,
510 doc=(
"A fractional measure of how much a value (like the exactLipschitz)" 511 "can change before it needs to be recalculated." 512 "This must be between 0 and 1."))
515 constraints = pexConfig.Field(dtype=str, default=
"1,+,S,M",
516 doc=(
"List of constraints to use for each object" 517 "(order does not matter)" 518 "Current options are all used by default:\n" 521 "1: normalized SED to unity" 522 "+: non-negative morphology"))
523 symmetryThresh = pexConfig.Field(dtype=float, default=1.0,
524 doc=(
"Strictness of symmetry, from" 525 "0 (no symmetry enforced) to" 526 "1 (perfect symmetry required)." 527 "If 'S' is not in `constraints`, this argument is ignored"))
528 l0Thresh = pexConfig.Field(dtype=float, default=np.nan,
529 doc=(
"L0 threshold. NaN results in no L0 penalty."))
530 l1Thresh = pexConfig.Field(dtype=float, default=np.nan,
531 doc=(
"L1 threshold. NaN results in no L1 penalty."))
532 tvxThresh = pexConfig.Field(dtype=float, default=np.nan,
533 doc=(
"Threshold for TV (total variation) constraint in the x-direction." 534 "NaN results in no TVx penalty."))
535 tvyThresh = pexConfig.Field(dtype=float, default=np.nan,
536 doc=(
"Threshold for TV (total variation) constraint in the y-direction." 537 "NaN results in no TVy penalty."))
540 useWeights = pexConfig.Field(dtype=bool, default=
False, doc=
"Use inverse variance as deblender weights")
541 bgScale = pexConfig.Field(
542 dtype=float, default=0.5,
543 doc=(
"Fraction of background RMS level to use as a" 544 "cutoff for defining the background of the image" 545 "This is used to initialize the model for each source" 546 "and to set the size of the bounding box for each source" 547 "every `refinementSkip` iteration."))
548 usePsfConvolution = pexConfig.Field(
549 dtype=bool, default=
True,
550 doc=(
"Whether or not to convolve the morphology with the" 551 "PSF in each band or use the same morphology in all bands"))
552 saveTemplates = pexConfig.Field(
553 dtype=bool, default=
True,
554 doc=
"Whether or not to save the SEDs and templates")
555 processSingles = pexConfig.Field(
556 dtype=bool, default=
False,
557 doc=
"Whether or not to process isolated sources in the deblender")
558 badMask = pexConfig.Field(
559 dtype=str, default=
"BAD,CR,NO_DATA,SAT,SUSPECT",
560 doc=
"Whether or not to process isolated sources in the deblender")
563 maxNumberOfPeaks = pexConfig.Field(
564 dtype=int, default=0,
565 doc=(
"Only deblend the brightest maxNumberOfPeaks peaks in the parent" 566 " (<= 0: unlimited)"))
567 maxFootprintArea = pexConfig.Field(
568 dtype=int, default=1000000,
569 doc=(
"Maximum area for footprints before they are ignored as large; " 570 "non-positive means no threshold applied"))
571 maxFootprintSize = pexConfig.Field(
572 dtype=int, default=0,
573 doc=(
"Maximum linear dimension for footprints before they are ignored " 574 "as large; non-positive means no threshold applied"))
575 minFootprintAxisRatio = pexConfig.Field(
576 dtype=float, default=0.0,
577 doc=(
"Minimum axis ratio for footprints before they are ignored " 578 "as large; non-positive means no threshold applied"))
579 notDeblendedMask = pexConfig.Field(
580 dtype=str, default=
"NOT_DEBLENDED", optional=
True,
581 doc=
"Mask name for footprints not deblended, or None")
583 tinyFootprintSize = pexConfig.RangeField(
584 dtype=int, default=2, min=2, inclusiveMin=
True,
585 doc=(
'Footprints smaller in width or height than this value will ' 586 'be ignored; minimum of 2 due to PSF gradient calculation.'))
587 catchFailures = pexConfig.Field(
588 dtype=bool, default=
False,
589 doc=(
"If True, catch exceptions thrown by the deblender, log them, " 590 "and set a flag on the parent, instead of letting them propagate up"))
591 propagateAllPeaks = pexConfig.Field(dtype=bool, default=
False,
592 doc=(
'Guarantee that all peaks produce a child source.'))
593 maskPlanes = pexConfig.ListField(dtype=str, default=[
"SAT",
"INTRP",
"NO_DATA"],
594 doc=
"Mask planes to ignore when performing statistics")
595 maskLimits = pexConfig.DictField(
599 doc=(
"Mask planes with the corresponding limit on the fraction of masked pixels. " 600 "Sources violating this limit will not be deblended."),
603 edgeHandling = pexConfig.ChoiceField(
604 doc=
'What to do when a peak to be deblended is close to the edge of the image',
605 dtype=str, default=
'noclip',
607 'clip':
'Clip the template at the edge AND the mirror of the edge.',
608 'ramp':
'Ramp down flux at the image edge by the PSF',
609 'noclip':
'Ignore the edge when building the symmetric template.',
613 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=
False,
614 doc=
"Apply a smoothing filter to all of the template images")
615 medianFilterHalfsize = pexConfig.Field(dtype=float, default=2,
616 doc=(
'Half size of the median smoothing filter'))
617 clipFootprintToNonzero = pexConfig.Field(dtype=bool, default=
False,
618 doc=(
"Clip non-zero spans in the footprints"))
620 conserveFlux = pexConfig.Field(dtype=bool, default=
True,
621 doc=(
"Reapportion flux to the footprints so that flux is conserved"))
622 weightTemplates = pexConfig.Field(
623 dtype=bool, default=
False,
624 doc=(
"If true, a least-squares fit of the templates will be done to the " 625 "full image. The templates will be re-weighted based on this fit."))
626 strayFluxToPointSources = pexConfig.ChoiceField(
627 doc=
'When the deblender should attribute stray flux to point sources',
628 dtype=str, default=
'necessary',
630 'necessary':
'When there is not an extended object in the footprint',
632 'never': (
'Never; stray flux will not be attributed to any deblended child ' 633 'if the deblender thinks all peaks look like point sources'),
637 assignStrayFlux = pexConfig.Field(dtype=bool, default=
True,
638 doc=
'Assign stray flux (not claimed by any child in the deblender) ' 639 'to deblend children.')
641 strayFluxRule = pexConfig.ChoiceField(
642 doc=
'How to split flux among peaks',
643 dtype=str, default=
'trim',
645 'r-to-peak':
'~ 1/(1+R^2) to the peak',
646 'r-to-footprint': (
'~ 1/(1+R^2) to the closest pixel in the footprint. ' 647 'CAUTION: this can be computationally expensive on large footprints!'),
648 'nearest-footprint': (
'Assign 100% to the nearest footprint (using L-1 norm aka ' 649 'Manhattan distance)'),
650 'trim': (
'Shrink the parent footprint to pixels that are not assigned to children')
654 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
655 doc=(
'When splitting stray flux, clip fractions below ' 656 'this value to zero.'))
657 getTemplateSum = pexConfig.Field(dtype=bool, default=
False,
658 doc=(
"As part of the flux calculation, the sum of the templates is" 659 "calculated. If 'getTemplateSum==True' then the sum of the" 660 "templates is stored in the result (a 'PerFootprint')."))
664 """MultibandDeblendTask 666 Split blended sources into individual sources. 668 This task has no return value; it only modifies the SourceCatalog in-place. 670 ConfigClass = MultibandDeblendConfig
671 _DefaultName =
"multibandDeblend" 673 def __init__(self, schema, peakSchema=None, **kwargs):
674 """Create the task, adding necessary fields to the given schema. 678 schema: `lsst.afw.table.schema.schema.Schema` 679 Schema object for measurement fields; will be modified in-place. 680 peakSchema: `lsst.afw.table.schema.schema.Schema` 681 Schema of Footprint Peaks that will be passed to the deblender. 682 Any fields beyond the PeakTable minimal schema will be transferred 683 to the main source Schema. If None, no fields will be transferred 686 Names of the filters used for the eposures. This is needed to store the SED as a field 688 Passed to Task.__init__. 692 pipeBase.Task.__init__(self, **kwargs)
693 if not self.config.conserveFlux
and not self.config.saveTemplates:
694 raise ValueError(
"Either `conserveFlux` or `saveTemplates` must be True")
696 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
697 if peakSchema
is None:
703 for item
in peakSchema:
704 if item.key
not in peakMinimalSchema:
710 schema.addField(item.field)
711 assert schema == self.
peakSchemaMapper.getOutputSchema(),
"Logic bug mapping schemas" 718 config = scarlet.config.Config(
719 center_min_dist=self.config.minTranslation,
720 edge_flux_thresh=self.config.edgeFluxThresh,
721 exact_lipschitz=self.config.exactLipschitz,
722 refine_skip=self.config.refinementSkip,
723 slack=self.config.stepSlack,
725 if self.config.translationMethod !=
"default":
726 err =
"Currently the only supported translationMethod is 'default', you entered '{0}'" 727 raise NotImplementedError(err.format(self.config.translationMethod))
732 _constraints = self.config.constraints.split(
",")
733 if (sorted(_constraints) != [
'+',
'1',
'M',
'S']
or 734 ~np.isnan(self.config.l0Thresh)
or 735 ~np.isnan(self.config.l1Thresh)):
737 "+": scarlet.constraint.PositivityConstraint,
738 "1": scarlet.constraint.SimpleConstraint,
739 "M": scarlet.constraint.DirectMonotonicityConstraint(use_nearest=
False),
740 "S": scarlet.constraint.DirectSymmetryConstraint(sigma=self.config.symmetryThresh)
742 for c
in _constraints:
743 if constraints
is None:
744 constraints = [constraintDict[c]]
746 constraints += [constraintDict[c]]
747 if constraints
is None:
748 constraints = scarlet.constraint.MinimalConstraint()
749 if ~np.isnan(self.config.l0Thresh):
750 constraints += [scarlet.constraint.L0Constraint(self.config.l0Thresh)]
751 if ~np.isnan(self.config.l1Thresh):
752 constraints += [scarlet.constraint.L1Constraint(self.config.l1Thresh)]
753 if ~np.isnan(self.config.tvxThresh):
754 constraints += [scarlet.constraint.TVxConstraint(self.config.tvxThresh)]
755 if ~np.isnan(self.config.tvyThresh):
756 constraints += [scarlet.constraint.TVyConstraint(self.config.tvyThresh)]
759 plugins.buildMultibandTemplates,
760 useWeights=self.config.useWeights,
761 usePsf=self.config.usePsfConvolution,
762 constraints=constraints,
764 maxIter=self.config.maxIter,
765 bgScale=self.config.bgScale,
766 relativeError=self.config.relativeError,
767 badMask=self.config.badMask.split(
","),
773 if self.config.edgeHandling ==
'ramp':
775 if self.config.medianSmoothTemplate:
777 plugins.medianSmoothTemplates,
778 medianFilterHalfsize=self.config.medianFilterHalfsize))
779 if self.config.clipFootprintToNonzero:
781 if self.config.conserveFlux:
782 if self.config.weightTemplates:
785 plugins.apportionFlux,
786 clipStrayFluxFraction=self.config.clipStrayFluxFraction,
787 assignStrayFlux=self.config.assignStrayFlux,
788 strayFluxAssignment=self.config.strayFluxRule,
789 strayFluxToPointSources=self.config.strayFluxToPointSources,
790 getTemplateSum=self.config.getTemplateSum))
792 def _addSchemaKeys(self, schema):
793 """Add deblender specific keys to the schema 795 self.
runtimeKey = schema.addField(
'runtime', type=np.float32, doc=
'runtime in ms')
797 self.
nChildKey = schema.addField(
'deblend_nChild', type=np.int32,
798 doc=
'Number of children this object has (defaults to 0)')
799 self.
psfKey = schema.addField(
'deblend_deblendedAsPsf', type=
'Flag',
800 doc=
'Deblender thought this source looked like a PSF')
802 doc=
'Source had too many peaks; ' 803 'only the brightest were included')
804 self.
tooBigKey = schema.addField(
'deblend_parentTooBig', type=
'Flag',
805 doc=
'Parent footprint covered too many pixels')
806 self.
maskedKey = schema.addField(
'deblend_masked', type=
'Flag',
807 doc=
'Parent footprint was predominantly masked')
809 doc=
"Deblending failed on source")
812 doc=
"Deblender skipped this source")
816 self.
psfCenterKey = afwTable.Point2DKey.addFields(schema,
'deblend_psfCenter',
817 'If deblended-as-psf, the PSF centroid',
"pixel")
818 self.
psfFluxKey = schema.addField(
'deblend_psf_instFlux', type=
'D',
819 doc=
'If deblended-as-psf, the instrumental PSF flux', units=
'count')
821 'deblend_rampedTemplate', type=
'Flag',
822 doc=(
'This source was near an image edge and the deblender used ' 823 '"ramp" edge-handling.'))
826 'deblend_patchedTemplate', type=
'Flag',
827 doc=(
'This source was near an image edge and the deblender used ' 828 '"patched" edge-handling.'))
831 'deblend_hasStrayFlux', type=
'Flag',
832 doc=(
'This source was assigned some stray flux'))
834 self.log.trace(
'Added keys to schema: %s',
", ".join(str(x)
for x
in (
839 def run(self, mExposure, mergedSources):
840 """Get the psf from each exposure and then run deblend(). 844 mExposure: `MultibandExposure` 845 The exposures should be co-added images of the same 846 shape and region of the sky. 847 mergedSources: `SourceCatalog` 848 The merged `SourceCatalog` that contains parent footprints 849 to (potentially) deblend. 853 fluxCatalogs: dict or None 854 Keys are the names of the filters and the values are 855 `lsst.afw.table.source.source.SourceCatalog`'s. 856 These are the flux-conserved catalogs with heavy footprints with 857 the image data weighted by the multiband templates. 858 If `self.config.conserveFlux` is `False`, then this item will be None 859 templateCatalogs: dict or None 860 Keys are the names of the filters and the values are 861 `lsst.afw.table.source.source.SourceCatalog`'s. 862 These are catalogs with heavy footprints that are the templates 863 created by the multiband templates. 864 If `self.config.saveTemplates` is `False`, then this item will be None 866 psfs = {f: mExposure[f].getPsf()
for f
in mExposure.filters}
867 return self.
deblend(mExposure, mergedSources, psfs)
869 def _getPsfFwhm(self, psf, bbox):
870 return psf.computeShape().getDeterminantRadius() * 2.35
872 def _addChild(self, parentId, peak, sources, heavy):
873 """Add a child to a catalog 875 This creates a new child in the source catalog, 876 assigning it a parent id, adding a footprint, 877 and setting all appropriate flags based on the 880 assert len(heavy.getPeaks()) == 1
881 src = sources.addNew()
883 src.setParent(parentId)
884 src.setFootprint(heavy)
885 src.set(self.
psfKey, peak.deblendedAsPsf)
894 """Deblend a data cube of multiband images 898 mExposure: `MultibandExposure` 899 The exposures should be co-added images of the same 900 shape and region of the sky. 901 sources: `SourceCatalog` 902 The merged `SourceCatalog` that contains parent footprints 903 to (potentially) deblend. 905 Keys are the names of the filters 906 (should be the same as `mExposure.filters`) 907 and the values are the PSFs in each band. 911 fluxCatalogs: dict or None 912 Keys are the names of the filters and the values are 913 `lsst.afw.table.source.source.SourceCatalog`'s. 914 These are the flux-conserved catalogs with heavy footprints with 915 the image data weighted by the multiband templates. 916 If `self.config.conserveFlux` is `False`, then this item will be None 917 templateCatalogs: dict or None 918 Keys are the names of the filters and the values are 919 `lsst.afw.table.source.source.SourceCatalog`'s. 920 These are catalogs with heavy footprints that are the templates 921 created by the multiband templates. 922 If `self.config.saveTemplates` is `False`, then this item will be None 926 if tuple(psfs.keys()) != mExposure.filters:
927 msg =
"PSF keys must be the same as mExposure.filters ({0}), got {1}" 928 raise ValueError(msg.format(mExposure.filters, psfs.keys()))
930 filters = mExposure.filters
931 mMaskedImage = afwImage.MultibandMaskedImage(filters=mExposure.filters, image=mExposure.image,
932 mask=mExposure.mask, variance=mExposure.variance)
933 self.log.info(
"Deblending {0} sources in {1} exposures".format(len(sources), len(mExposure)))
938 exposure = mExposure[f]
939 mi = exposure.getMaskedImage()
940 statsCtrl = afwMath.StatisticsControl()
941 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
942 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
943 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
944 self.log.trace(
'Exposure {0}, sigma1: {1}'.format(f, sigma1))
948 if self.config.conserveFlux:
951 _catalog = afwTable.SourceCatalog(sources.table.clone())
952 _catalog.extend(sources)
953 fluxCatalogs[f] = _catalog
956 if self.config.saveTemplates:
957 templateCatalogs = {}
959 _catalog = afwTable.SourceCatalog(sources.table.clone())
960 _catalog.extend(sources)
961 templateCatalogs[f] = _catalog
963 templateCatalogs =
None 967 for pk, src
in enumerate(sources):
968 foot = src.getFootprint()
969 logger.info(
"id: {0}".format(src[
"id"]))
970 peaks = foot.getPeaks()
977 if len(peaks) < 2
and not self.config.processSingles:
979 if self.config.saveTemplates:
980 templateCatalogs[f][pk].set(self.
runtimeKey, 0)
981 if self.config.conserveFlux:
986 self.
skipParent(src, [mi.getMask()
for mi
in mMaskedImage])
987 self.log.warn(
'Parent %i: skipping large footprint (area: %i)',
988 int(src.getId()), int(foot.getArea()))
990 if self.
isMasked(foot, exposure.getMaskedImage().getMask()):
993 self.log.trace(
'Parent %i: skipping masked footprint (area: %i)',
994 int(src.getId()), int(foot.getArea()))
996 if len(peaks) > self.config.maxNumberOfPeaks:
998 msg =
'Parent {0}: Too many peaks, using the first {1} peaks' 999 self.log.trace(msg.format(int(src.getId()), self.config.maxNumberOfPeaks))
1002 bbox = foot.getBBox()
1003 psf_fwhms = {f: self.
_getPsfFwhm(psf, bbox)
for f, psf
in psfs.items()}
1004 self.log.trace(
'Parent %i: deblending %i peaks', int(src.getId()), len(peaks))
1011 images = mMaskedImage[:, bbox]
1012 psf_list = [psfs[f]
for f
in filters]
1013 fwhm_list = [psf_fwhms[f]
for f
in filters]
1014 avgNoise = [sigmas[f]
for f
in filters]
1018 mMaskedImage=images,
1022 maxNumberOfPeaks=self.config.maxNumberOfPeaks)
1024 runtime = (tf-t0)*1000
1029 except Exception
as e:
1030 if self.config.catchFailures:
1031 self.log.warn(
"Unable to deblend source %d: %s" % (src.getId(), e))
1035 traceback.print_exc()
1041 templateParents = {}
1043 parentId = src.getId()
1045 if self.config.saveTemplates:
1046 templateParents[f] = templateCatalogs[f][pk]
1047 templateParents[f].set(self.
runtimeKey, runtime)
1048 if self.config.conserveFlux:
1049 fluxParents[f] = fluxCatalogs[f][pk]
1056 for j, multiPeak
in enumerate(result.peaks):
1057 heavy = {f: peak.getFluxPortion()
for f, peak
in multiPeak.deblendedPeaks.items()}
1058 no_flux = all([v
is None for v
in heavy.values()])
1059 skip_peak = all([peak.skip
for peak
in multiPeak.deblendedPeaks.values()])
1060 if no_flux
or skip_peak:
1062 if not self.config.propagateAllPeaks:
1067 msg =
"Peak at {0} failed deblending. Using minimal default info for child." 1068 self.log.trace(msg.format(multiPeak.x, multiPeak.y))
1071 pfoot = afwDet.Footprint(foot)
1072 peakList = pfoot.getPeaks()
1074 pfoot.addPeak(multiPeak.x, multiPeak.y, 0)
1075 zeroMimg = afwImage.MaskedImageF(pfoot.getBBox())
1077 heavy[f] = afwDet.makeHeavyFootprint(pfoot, zeroMimg)
1083 if len(heavy[f].getPeaks()) != 1:
1084 err =
"Heavy footprint should have a single peak, got {0}" 1085 raise ValueError(err.format(len(heavy[f].getPeaks())))
1086 peak = multiPeak.deblendedPeaks[f]
1087 if self.config.saveTemplates:
1088 cat = templateCatalogs[f]
1089 tfoot = peak.templateFootprint
1090 timg = afwImage.MaskedImageF(peak.templateImage)
1091 tHeavy = afwDet.makeHeavyFootprint(tfoot, timg)
1092 child = self.
_addChild(parentId, peak, cat, tHeavy)
1094 child.setId(src.getId())
1097 templateSpans[f] = templateSpans[f].union(tHeavy.getSpans())
1098 if self.config.conserveFlux:
1099 cat = fluxCatalogs[f]
1100 child = self.
_addChild(parentId, peak, cat, heavy[f])
1102 child.setId(src.getId())
1105 fluxSpans[f] = fluxSpans[f].union(heavy[f].getSpans())
1114 if self.config.saveTemplates:
1115 templateParents[f].set(self.
nChildKey, nchild)
1116 templateParents[f].getFootprint().setSpans(templateSpans[f])
1117 if self.config.conserveFlux:
1118 fluxParents[f].set(self.
nChildKey, nchild)
1119 fluxParents[f].getFootprint().setSpans(fluxSpans[f])
1122 pk, npre, foot, psfs, psf_fwhms, sigmas, result)
1124 if fluxCatalogs
is not None:
1125 n1 = len(list(fluxCatalogs.values())[0])
1127 n1 = len(list(templateCatalogs.values())[0])
1128 self.log.info(
'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' 1129 % (n0, nparents, n1-n0, n1))
1130 return fluxCatalogs, templateCatalogs
1136 pk, npre, fp, psfs, psf_fwhms, sigmas, result):
1140 """Returns whether a Footprint is large 1142 'Large' is defined by thresholds on the area, size and axis ratio. 1143 These may be disabled independently by configuring them to be non-positive. 1145 This is principally intended to get rid of satellite streaks, which the 1146 deblender or other downstream processing can have trouble dealing with 1147 (e.g., multiple large HeavyFootprints can chew up memory). 1149 if self.config.maxFootprintArea > 0
and footprint.getArea() > self.config.maxFootprintArea:
1151 if self.config.maxFootprintSize > 0:
1152 bbox = footprint.getBBox()
1153 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
1155 if self.config.minFootprintAxisRatio > 0:
1156 axes = afwEll.Axes(footprint.getShape())
1157 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
1162 """Returns whether the footprint violates the mask limits""" 1163 size = float(footprint.getArea())
1164 for maskName, limit
in self.config.maskLimits.items():
1165 maskVal = mask.getPlaneBitMask(maskName)
1166 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal)
1167 if (size - unmaskedSpan.getArea())/size > limit:
1172 """Indicate that the parent source is not being deblended 1174 We set the appropriate flags and masks for each exposure. 1178 source: `lsst.afw.table.source.source.SourceRecord` 1179 The source to flag as skipped 1180 masks: list of `lsst.afw.image.MaskX` 1181 The mask in each band to update with the non-detection 1183 fp = source.getFootprint()
1185 source.set(self.
nChildKey, len(fp.getPeaks()))
1186 if self.config.notDeblendedMask:
1188 mask.addMaskPlane(self.config.notDeblendedMask)
1189 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
def newDeblend(debPlugins, footprint, mMaskedImage, psfs, psfFwhms, log=None, verbose=False, avgNoise=None, maxNumberOfPeaks=0)
def _addChild(self, parentId, peak, sources, heavy)
def isLargeFootprint(self, footprint)
def isLargeFootprint(self, footprint)
deblendPatchedTemplateKey
def postSingleDeblendHook(self, exposures, fluxCatalogs, templateCatalogs, pk, npre, fp, psfs, psf_fwhms, sigmas, result)
def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
def _getPsfFwhm(self, psf, bbox)
def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
def run(self, exposure, sources)
Get the psf from the provided exposure and then run deblend().
def __init__(self, schema, peakSchema=None, kwargs)
def _getPsfFwhm(self, psf, bbox)
def _addSchemaKeys(self, schema)
def preSingleDeblendHook(self, exposures, sources, pk, fp, psfs, psf_fwhms, sigmas)
static Log getLogger(std::string const &loggername)
def addSchemaKeys(self, schema)
Split blended sources into individual sources.
def skipParent(self, source, mask)
def deblend(self, exposure, srcs, psf)
Deblend.
def skipParent(self, source, masks)
def __init__(self, schema, peakSchema=None, kwargs)
Create the task, adding necessary fields to the given schema.
def deblend(self, mExposure, sources, psfs)
def isMasked(self, footprint, mask)
deblendPatchedTemplateKey
def isMasked(self, footprint, mask)
def run(self, mExposure, mergedSources)