Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%
447 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:44 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:44 +0000
1# This file is part of meas_extensions_scarlet.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from dataclasses import dataclass
23from functools import partial
24import logging
25import numpy as np
26import scarlet
27from scarlet.psf import ImagePSF, GaussianPSF
28from scarlet import Blend, Frame, Observation
29from scarlet.renderer import ConvolutionRenderer
30from scarlet.detect import get_detect_wavelets
31from scarlet.initialization import init_all_sources
32from scarlet import lite
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36from lsst.geom import Point2I, Point2D
37import lsst.afw.geom.ellipses as afwEll
38import lsst.afw.image as afwImage
39import lsst.afw.detection as afwDet
40import lsst.afw.table as afwTable
41from lsst.utils.logging import PeriodicLogger
42from lsst.utils.timer import timeMethod
44from .source import bboxToScarletBox
45from .io import ScarletModelData, scarletToData, scarletLiteToData
47# Scarlet and proxmin have a different definition of log levels than the stack,
48# so even "warnings" occur far more often than we would like.
49# So for now we only display scarlet and proxmin errors, as all other
50# scarlet outputs would be considered "TRACE" by our standards.
51scarletLogger = logging.getLogger("scarlet")
52scarletLogger.setLevel(logging.ERROR)
53proxminLogger = logging.getLogger("proxmin")
54proxminLogger.setLevel(logging.ERROR)
56__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"]
58logger = logging.getLogger(__name__)
61class IncompleteDataError(Exception):
62 """The PSF could not be computed due to incomplete data
63 """
64 pass
67class ScarletGradientError(Exception):
68 """An error occurred during optimization
70 This error occurs when the optimizer encounters
71 a NaN value while calculating the gradient.
72 """
73 def __init__(self, iterations, sources):
74 self.iterations = iterations
75 self.sources = sources
76 msg = ("ScalarGradientError in iteration {0}. "
77 "NaN values introduced in sources {1}")
78 self.message = msg.format(iterations, sources)
80 def __str__(self):
81 return self.message
84def _checkBlendConvergence(blend, f_rel):
85 """Check whether or not a blend has converged
86 """
87 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1])
88 convergence = f_rel * np.abs(blend.loss[-1])
89 return deltaLoss < convergence
92def isPseudoSource(source, pseudoColumns):
93 """Check if a source is a pseudo source.
95 This is mostly for skipping sky objects,
96 but any other column can also be added to disable
97 deblending on a parent or individual source when
98 set to `True`.
100 Parameters
101 ----------
102 source : `lsst.afw.table.source.source.SourceRecord`
103 The source to check for the pseudo bit.
104 pseudoColumns : `list` of `str`
105 A list of columns to check for pseudo sources.
106 """
107 isPseudo = False
108 for col in pseudoColumns:
109 try:
110 isPseudo |= source[col]
111 except KeyError:
112 pass
113 return isPseudo
116def deblend(mExposure, footprint, config, spectrumInit):
117 """Deblend a parent footprint
119 Parameters
120 ----------
121 mExposure : `lsst.image.MultibandExposure`
122 The multiband exposure containing the image,
123 mask, and variance data.
124 footprint : `lsst.detection.Footprint`
125 The footprint of the parent to deblend.
126 config : `ScarletDeblendConfig`
127 Configuration of the deblending task.
128 spectrumInit : `bool`
129 Whether or not to initialize the spectrum.
131 Returns
132 -------
133 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData`
134 The persistable representation of a `scarlet.Blend`.
135 skipped : `list` of `int`
136 The indices of any children that failed to initialize
137 and were skipped.
138 spectrumInit : `bool`
139 Whether or not all of the sources were initialized by jointly
140 fitting their SED's. This provides a better initialization
141 but created memory issues when a blend is too large or
142 contains too many sources.
143 """
144 # Extract coordinates from each MultiColorPeak
145 bbox = footprint.getBBox()
147 # Create the data array from the masked images
148 images = mExposure.image[:, bbox].array
150 # Use the inverse variance as the weights
151 if config.useWeights:
152 weights = 1/mExposure.variance[:, bbox].array
153 else:
154 weights = np.ones_like(images)
155 badPixels = mExposure.mask.getPlaneBitMask(config.badMask)
156 mask = mExposure.mask[:, bbox].array & badPixels
157 weights[mask > 0] = 0
159 # Mask out the pixels outside the footprint
160 weights *= footprint.spans.asArray()
162 psfCenter = footprint.getCentroid()
163 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32)
164 psfs = ImagePSF(psfs)
165 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters))
167 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters)
168 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters)
169 if config.convolutionType == "fft":
170 observation.match(frame)
171 elif config.convolutionType == "real":
172 renderer = ConvolutionRenderer(observation, frame, convolution_type="real")
173 observation.match(frame, renderer=renderer)
174 else:
175 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType))
177 assert(config.sourceModel in ["single", "double", "compact", "fit"])
179 # Set the appropriate number of components
180 if config.sourceModel == "single":
181 maxComponents = 1
182 elif config.sourceModel == "double":
183 maxComponents = 2
184 elif config.sourceModel == "compact":
185 maxComponents = 0
186 elif config.sourceModel == "point":
187 raise NotImplementedError("Point source photometry is currently not implemented")
188 elif config.sourceModel == "fit":
189 # It is likely in the future that there will be some heuristic
190 # used to determine what type of model to use for each source,
191 # but that has not yet been implemented (see DM-22551)
192 raise NotImplementedError("sourceModel 'fit' has not been implemented yet")
194 # Convert the centers to pixel coordinates
195 xmin = bbox.getMinX()
196 ymin = bbox.getMinY()
197 centers = [
198 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
199 for peak in footprint.peaks
200 if not isPseudoSource(peak, config.pseudoColumns)
201 ]
203 # Only deblend sources that can be initialized
204 sources, skipped = init_all_sources(
205 frame=frame,
206 centers=centers,
207 observations=observation,
208 thresh=config.morphThresh,
209 max_components=maxComponents,
210 min_snr=config.minSNR,
211 shifting=False,
212 fallback=config.fallback,
213 silent=config.catchFailures,
214 set_spectra=spectrumInit,
215 )
217 # Attach the peak to all of the initialized sources
218 srcIndex = 0
219 for k, center in enumerate(centers):
220 if k not in skipped:
221 # This is just to make sure that there isn't a coding bug
222 assert np.all(sources[srcIndex].center == center)
223 # Store the record for the peak with the appropriate source
224 sources[srcIndex].detectedPeak = footprint.peaks[k]
225 srcIndex += 1
227 # Create the blend and attempt to optimize it
228 blend = Blend(sources, observation)
229 try:
230 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError)
231 except ArithmeticError:
232 # This occurs when a gradient update produces a NaN value
233 # This is usually due to a source initialized with a
234 # negative SED or no flux, often because the peak
235 # is a noise fluctuation in one band and not a real source.
236 iterations = len(blend.loss)
237 failedSources = []
238 for k, src in enumerate(sources):
239 if np.any(~np.isfinite(src.get_model())):
240 failedSources.append(k)
241 raise ScarletGradientError(iterations, failedSources)
243 # Store the location of the PSF center for storage
244 blend.psfCenter = (psfCenter.x, psfCenter.y)
246 return blend, skipped
249def buildLiteObservation(
250 modelPsf,
251 psfCenter,
252 mExposure,
253 footprint=None,
254 badPixelMasks=None,
255 useWeights=True,
256 convolutionType="real",
257):
258 """Generate a LiteObservation from a set of parameters.
260 Make the generation and reconstruction of a scarlet model consistent
261 by building a `LiteObservation` from a set of parameters.
263 Parameters
264 ----------
265 modelPsf : `numpy.ndarray`
266 The 2D model of the PSF in the partially deconvolved space.
267 psfCenter : `tuple` or `Point2I` or `Point2D`
268 The location `(x, y)` used as the center of the PSF.
269 mExposure : `lsst.afw.image.multiband.MultibandExposure`
270 The multi-band exposure that the model represents.
271 If `mExposure` is `None` then no image, variance, or weights are
272 attached to the observation.
273 footprint : `lsst.afw.detection.Footprint`
274 The footprint that is being fit.
275 If `Footprint` is `None` then the weights are not updated to mask
276 out pixels not contained in the footprint.
277 badPixelMasks : `list` of `str`
278 The keys from the bit mask plane used to mask out pixels
279 during the fit.
280 If `badPixelMasks` is `None` then the default values from
281 `ScarletDeblendConfig.badMask` is used.
282 useWeights : `bool`
283 Whether or not fitting should use inverse variance weights to
284 calculate the log-likelihood.
285 convolutionType : `str`
286 The type of convolution to use (either "real" or "fft").
287 When reconstructing an image it is advised to use "real" to avoid
288 polluting the footprint with
290 Returns
291 -------
292 observation : `scarlet.lite.LiteObservation`
293 The observation constructed from the input parameters.
294 """
295 # Initialize the observed PSFs
296 if not isinstance(psfCenter, Point2D):
297 psfCenter = Point2D(*psfCenter)
298 psfModels = mExposure.computePsfKernelImage(psfCenter)
300 # Use the inverse variance as the weights
301 if useWeights:
302 weights = 1/mExposure.variance.array
303 else:
304 # Mask out bad pixels
305 weights = np.ones_like(mExposure.image.array)
306 if badPixelMasks is None:
307 badPixelMasks = ScarletDeblendConfig().badMask
308 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks)
309 mask = mExposure.mask.array & badPixels
310 weights[mask > 0] = 0
312 if footprint is not None:
313 # Mask out the pixels outside the footprint
314 weights *= footprint.spans.asArray()
316 return lite.LiteObservation(
317 images=mExposure.image.array,
318 variance=mExposure.variance.array,
319 weights=weights,
320 psfs=psfModels,
321 model_psf=modelPsf[None, :, :],
322 convolution_mode=convolutionType,
323 )
326def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None):
327 """Deblend a parent footprint
329 Parameters
330 ----------
331 mExposure : `lsst.image.MultibandExposure`
332 - The multiband exposure containing the image,
333 mask, and variance data
334 footprint : `lsst.detection.Footprint`
335 - The footprint of the parent to deblend
336 config : `ScarletDeblendConfig`
337 - Configuration of the deblending task
338 """
339 # Extract coordinates from each MultiColorPeak
340 bbox = footprint.getBBox()
341 psfCenter = footprint.getCentroid()
343 observation = buildLiteObservation(
344 modelPsf=modelPsf,
345 psfCenter=psfCenter,
346 mExposure=mExposure[:, bbox],
347 footprint=footprint,
348 badPixelMasks=config.badMask,
349 useWeights=config.useWeights,
350 convolutionType=config.convolutionType,
351 )
353 # Convert the centers to pixel coordinates
354 xmin = bbox.getMinX()
355 ymin = bbox.getMinY()
356 centers = [
357 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
358 for peak in footprint.peaks
359 if not isPseudoSource(peak, config.pseudoColumns)
360 ]
362 # Initialize the sources
363 if config.morphImage == "chi2":
364 sources = lite.init_all_sources_main(
365 observation,
366 centers,
367 min_snr=config.minSNR,
368 thresh=config.morphThresh,
369 )
370 elif config.morphImage == "wavelet":
371 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin())
372 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)]
373 sources = lite.init_all_sources_wavelets(
374 observation,
375 centers,
376 use_psf=False,
377 wavelets=_wavelets,
378 min_snr=config.minSNR,
379 )
380 else:
381 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.")
383 # Set the optimizer
384 if config.optimizer == "adaprox":
385 parameterization = partial(
386 lite.init_adaprox_component,
387 bg_thresh=config.backgroundThresh,
388 max_prox_iter=config.maxProxIter,
389 )
390 elif config.optimizer == "fista":
391 parameterization = partial(
392 lite.init_fista_component,
393 bg_thresh=config.backgroundThresh,
394 )
395 else:
396 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.")
397 sources = lite.parameterize_sources(sources, observation, parameterization)
399 # Attach the peak to all of the initialized sources
400 for k, center in enumerate(centers):
401 # This is just to make sure that there isn't a coding bug
402 if len(sources[k].components) > 0 and np.any(sources[k].center != center):
403 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}")
404 # Store the record for the peak with the appropriate source
405 sources[k].detectedPeak = footprint.peaks[k]
407 blend = lite.LiteBlend(sources, observation)
409 # Initialize each source with its best fit spectrum
410 if spectrumInit:
411 blend.fit_spectra()
413 # Set the sources that could not be initialized and were skipped
414 skipped = [src for src in sources if src.is_null]
416 blend.fit(
417 max_iter=config.maxIter,
418 e_rel=config.relativeError,
419 min_iter=config.minIter,
420 reweight=False,
421 )
423 # Store the location of the PSF center for storage
424 blend.psfCenter = (psfCenter.x, psfCenter.y)
426 return blend, skipped
429@dataclass
430class DeblenderMetrics:
431 """Metrics and measurements made on single sources.
433 Store deblender metrics to be added as attributes to a scarlet source
434 before it is converted into a `SourceRecord`.
435 When DM-34414 is finished this class will be eliminated and the metrics
436 will be added to the schema using a pipeline task that calculates them
437 from the stored deconvolved models.
439 All of the parameters are one dimensional numpy arrays,
440 with an element for each band in the observed images.
442 `maxOverlap` is useful as a metric for determining how blended a source
443 is because if it only overlaps with other sources at or below
444 the noise level, it is likely to be a mostly isolated source
445 in the deconvolved model frame.
447 `fluxOverlapFraction` is potentially more useful than the canonical
448 "blendedness" (or purity) metric because it accounts for potential
449 biases created during deblending by not weighting the overlapping
450 flux with the flux of this sources model.
452 Attributes
453 ----------
454 maxOverlap : `numpy.ndarray`
455 The maximum overlap that the source has with its neighbors in
456 a single pixel.
457 fluxOverlap : `numpy.ndarray`
458 The total flux from neighbors overlapping with the current source.
459 fluxOverlapFraction : `numpy.ndarray`
460 The fraction of `flux from neighbors/source flux` for a
461 given source within the source's footprint.
462 blendedness : `numpy.ndarray`
463 The metric for determining how blended a source is using the
464 Bosch et al. 2018 metric for "blendedness." Note that some
465 surveys use the term "purity," which is `1-blendedness`.
466 """
467 maxOverlap: np.array
468 fluxOverlap: np.array
469 fluxOverlapFraction: np.array
470 blendedness: np.array
473def setDeblenderMetrics(blend):
474 """Set metrics that can be used to evalute the deblender accuracy
476 This function calculates the `DeblenderMetrics` for each source in the
477 blend, and assigns it to that sources `metrics` property in place.
479 Parameters
480 ----------
481 blend : `scarlet.lite.Blend`
482 The blend containing the sources to measure.
483 """
484 # Store the full model of the scene for comparison
485 blendModel = blend.get_model()
486 for k, src in enumerate(blend.sources):
487 # Extract the source model in the full bounding box
488 model = src.get_model(bbox=blend.bbox)
489 # The footprint is the 2D array of non-zero pixels in each band
490 footprint = np.bitwise_or.reduce(model > 0, axis=0)
491 # Calculate the metrics.
492 # See `DeblenderMetrics` for a description of each metric.
493 neighborOverlap = (blendModel-model) * footprint[None, :, :]
494 maxOverlap = np.max(neighborOverlap, axis=(1, 2))
495 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2))
496 fluxModel = np.sum(model, axis=(1, 2))
497 fluxOverlapFraction = np.zeros((len(model), ), dtype=float)
498 isFinite = fluxModel > 0
499 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite]
500 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2))
501 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness)
504class ScarletDeblendConfig(pexConfig.Config):
505 """MultibandDeblendConfig
507 Configuration for the multiband deblender.
508 The parameters are organized by the parameter types, which are
509 - Stopping Criteria: Used to determine if the fit has converged
510 - Position Fitting Criteria: Used to fit the positions of the peaks
511 - Constraints: Used to apply constraints to the peaks and their components
512 - Other: Parameters that don't fit into the above categories
513 """
514 # Stopping Criteria
515 minIter = pexConfig.Field(dtype=int, default=1,
516 doc="Minimum number of iterations before the optimizer is allowed to stop.")
517 maxIter = pexConfig.Field(dtype=int, default=300,
518 doc=("Maximum number of iterations to deblend a single parent"))
519 relativeError = pexConfig.Field(dtype=float, default=1e-2,
520 doc=("Change in the loss function between iterations to exit fitter. "
521 "Typically this is `1e-2` if measurements will be made on the "
522 "flux re-distributed models and `1e-4` when making measurements "
523 "on the models themselves."))
525 # Constraints
526 morphThresh = pexConfig.Field(dtype=float, default=1,
527 doc="Fraction of background RMS a pixel must have"
528 "to be included in the initial morphology")
529 # Lite Parameters
530 # All of these parameters (except version) are only valid if version='lite'
531 version = pexConfig.ChoiceField(
532 dtype=str,
533 default="lite",
534 allowed={
535 "scarlet": "main scarlet version (likely to be deprecated soon)",
536 "lite": "Optimized version of scarlet for survey data from a single instrument",
537 },
538 doc="The version of scarlet to use.",
539 )
540 optimizer = pexConfig.ChoiceField(
541 dtype=str,
542 default="adaprox",
543 allowed={
544 "adaprox": "Proximal ADAM optimization",
545 "fista": "Accelerated proximal gradient method",
546 },
547 doc="The optimizer to use for fitting parameters and is only used when version='lite'",
548 )
549 morphImage = pexConfig.ChoiceField(
550 dtype=str,
551 default="chi2",
552 allowed={
553 "chi2": "Initialize sources on a chi^2 image made from all available bands",
554 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image",
555 },
556 doc="The type of image to use for initializing the morphology. "
557 "Must be either 'chi2' or 'wavelet'. "
558 )
559 backgroundThresh = pexConfig.Field(
560 dtype=float,
561 default=0.25,
562 doc="Fraction of background to use for a sparsity threshold. "
563 "This prevents sources from growing unrealistically outside "
564 "the parent footprint while still modeling flux correctly "
565 "for bright sources."
566 )
567 maxProxIter = pexConfig.Field(
568 dtype=int,
569 default=1,
570 doc="Maximum number of proximal operator iterations inside of each "
571 "iteration of the optimizer. "
572 "This config field is only used if version='lite' and optimizer='adaprox'."
573 )
574 waveletScales = pexConfig.Field(
575 dtype=int,
576 default=5,
577 doc="Number of wavelet scales to use for wavelet initialization. "
578 "This field is only used when `version`='lite' and `morphImage`='wavelet'."
579 )
581 # Other scarlet paremeters
582 useWeights = pexConfig.Field(
583 dtype=bool, default=True,
584 doc=("Whether or not use use inverse variance weighting."
585 "If `useWeights` is `False` then flat weights are used"))
586 modelPsfSize = pexConfig.Field(
587 dtype=int, default=11,
588 doc="Model PSF side length in pixels")
589 modelPsfSigma = pexConfig.Field(
590 dtype=float, default=0.8,
591 doc="Define sigma for the model frame PSF")
592 minSNR = pexConfig.Field(
593 dtype=float, default=50,
594 doc="Minimum Signal to noise to accept the source."
595 "Sources with lower flux will be initialized with the PSF but updated "
596 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).")
597 saveTemplates = pexConfig.Field(
598 dtype=bool, default=True,
599 doc="Whether or not to save the SEDs and templates")
600 processSingles = pexConfig.Field(
601 dtype=bool, default=True,
602 doc="Whether or not to process isolated sources in the deblender")
603 convolutionType = pexConfig.Field(
604 dtype=str, default="fft",
605 doc="Type of convolution to render the model to the observations.\n"
606 "- 'fft': perform convolutions in Fourier space\n"
607 "- 'real': peform convolutions in real space.")
608 sourceModel = pexConfig.Field(
609 dtype=str, default="double",
610 doc=("How to determine which model to use for sources, from\n"
611 "- 'single': use a single component for all sources\n"
612 "- 'double': use a bulge disk model for all sources\n"
613 "- 'compact': use a single component model, initialzed with a point source morphology, "
614 " for all sources\n"
615 "- 'point': use a point-source model for all sources\n"
616 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"),
617 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.",
618 )
619 setSpectra = pexConfig.Field(
620 dtype=bool, default=True,
621 doc="Whether or not to solve for the best-fit spectra during initialization. "
622 "This makes initialization slightly longer, as it requires a convolution "
623 "to set the optimal spectra, but results in a much better initial log-likelihood "
624 "and reduced total runtime, with convergence in fewer iterations."
625 "This option is only used when "
626 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.")
628 # Mask-plane restrictions
629 badMask = pexConfig.ListField(
630 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"],
631 doc="Whether or not to process isolated sources in the deblender")
632 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
633 doc="Mask planes to ignore when performing statistics")
634 maskLimits = pexConfig.DictField(
635 keytype=str,
636 itemtype=float,
637 default={},
638 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
639 "Sources violating this limit will not be deblended. "
640 "If the fraction is `0` then the limit is a single pixel."),
641 )
643 # Size restrictions
644 maxNumberOfPeaks = pexConfig.Field(
645 dtype=int, default=200,
646 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
647 " (<= 0: unlimited)"))
648 maxFootprintArea = pexConfig.Field(
649 dtype=int, default=100_000,
650 doc=("Maximum area for footprints before they are ignored as large; "
651 "non-positive means no threshold applied"))
652 maxAreaTimesPeaks = pexConfig.Field(
653 dtype=int, default=10_000_000,
654 doc=("Maximum rectangular footprint area * nPeaks in the footprint. "
655 "This was introduced in DM-33690 to prevent fields that are crowded or have a "
656 "LSB galaxy that causes memory intensive initialization in scarlet from dominating "
657 "the overall runtime and/or causing the task to run out of memory. "
658 "(<= 0: unlimited)")
659 )
660 maxFootprintSize = pexConfig.Field(
661 dtype=int, default=0,
662 doc=("Maximum linear dimension for footprints before they are ignored "
663 "as large; non-positive means no threshold applied"))
664 minFootprintAxisRatio = pexConfig.Field(
665 dtype=float, default=0.0,
666 doc=("Minimum axis ratio for footprints before they are ignored "
667 "as large; non-positive means no threshold applied"))
668 maxSpectrumCutoff = pexConfig.Field(
669 dtype=int, default=1_000_000,
670 doc=("Maximum number of pixels * number of sources in a blend. "
671 "This is different than `maxFootprintArea` because this isn't "
672 "the footprint area but the area of the bounding box that "
673 "contains the footprint, and is also multiplied by the number of"
674 "sources in the footprint. This prevents large skinny blends with "
675 "a high density of sources from running out of memory. "
676 "If `maxSpectrumCutoff == -1` then there is no cutoff.")
677 )
678 # Failure modes
679 fallback = pexConfig.Field(
680 dtype=bool, default=True,
681 doc="Whether or not to fallback to a smaller number of components if a source does not initialize"
682 )
683 notDeblendedMask = pexConfig.Field(
684 dtype=str, default="NOT_DEBLENDED", optional=True,
685 doc="Mask name for footprints not deblended, or None")
686 catchFailures = pexConfig.Field(
687 dtype=bool, default=True,
688 doc=("If True, catch exceptions thrown by the deblender, log them, "
689 "and set a flag on the parent, instead of letting them propagate up"))
691 # Other options
692 columnInheritance = pexConfig.DictField(
693 keytype=str, itemtype=str, default={
694 "deblend_nChild": "deblend_parentNChild",
695 "deblend_nPeaks": "deblend_parentNPeaks",
696 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag",
697 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag",
698 },
699 doc="Columns to pass from the parent to the child. "
700 "The key is the name of the column for the parent record, "
701 "the value is the name of the column to use for the child."
702 )
703 pseudoColumns = pexConfig.ListField(
704 dtype=str, default=['merge_peak_sky', 'sky_source'],
705 doc="Names of flags which should never be deblended."
706 )
708 # Testing options
709 # Some obs packages and ci packages run the full pipeline on a small
710 # subset of data to test that the pipeline is functioning properly.
711 # This is not meant as scientific validation, so it can be useful
712 # to only run on a small subset of the data that is large enough to
713 # test the desired pipeline features but not so long that the deblender
714 # is the tall pole in terms of execution times.
715 useCiLimits = pexConfig.Field(
716 dtype=bool, default=False,
717 doc="Limit the number of sources deblended for CI to prevent long build times")
718 ciDeblendChildRange = pexConfig.ListField(
719 dtype=int, default=[5, 10],
720 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
721 "If `useCiLimits==False` then this parameter is ignored.")
722 ciNumParentsToDeblend = pexConfig.Field(
723 dtype=int, default=10,
724 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
725 "within `ciDebledChildRange`. "
726 "If `useCiLimits==False` then this parameter is ignored.")
729class ScarletDeblendTask(pipeBase.Task):
730 """ScarletDeblendTask
732 Split blended sources into individual sources.
734 This task has no return value; it only modifies the SourceCatalog in-place.
735 """
736 ConfigClass = ScarletDeblendConfig
737 _DefaultName = "scarletDeblend"
739 def __init__(self, schema, peakSchema=None, **kwargs):
740 """Create the task, adding necessary fields to the given schema.
742 Parameters
743 ----------
744 schema : `lsst.afw.table.schema.schema.Schema`
745 Schema object for measurement fields; will be modified in-place.
746 peakSchema : `lsst.afw.table.schema.schema.Schema`
747 Schema of Footprint Peaks that will be passed to the deblender.
748 Any fields beyond the PeakTable minimal schema will be transferred
749 to the main source Schema. If None, no fields will be transferred
750 from the Peaks.
751 filters : list of str
752 Names of the filters used for the eposures. This is needed to store
753 the SED as a field
754 **kwargs
755 Passed to Task.__init__.
756 """
757 pipeBase.Task.__init__(self, **kwargs)
759 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
760 if peakSchema is None:
761 # In this case, the peakSchemaMapper will transfer nothing, but
762 # we'll still have one
763 # to simplify downstream code
764 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
765 else:
766 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
767 for item in peakSchema:
768 if item.key not in peakMinimalSchema:
769 self.peakSchemaMapper.addMapping(item.key, item.field)
770 # Because SchemaMapper makes a copy of the output schema
771 # you give its ctor, it isn't updating this Schema in
772 # place. That's probably a design flaw, but in the
773 # meantime, we'll keep that schema in sync with the
774 # peakSchemaMapper.getOutputSchema() manually, by adding
775 # the same fields to both.
776 schema.addField(item.field)
777 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
778 self._addSchemaKeys(schema)
779 self.schema = schema
780 self.toCopyFromParent = [item.key for item in self.schema
781 if item.field.getName().startswith("merge_footprint")]
783 def _addSchemaKeys(self, schema):
784 """Add deblender specific keys to the schema
785 """
786 # Parent (blend) fields
787 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms')
788 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge')
789 self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
790 doc='Number of children this object has (defaults to 0)')
791 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32,
792 doc="Number of initial peaks in the blend. "
793 "This includes peaks that may have been culled "
794 "during deblending or failed to deblend")
795 # Skipped flags
796 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
797 doc="Deblender skipped this source")
798 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag',
799 doc='The source has only a single peak '
800 'and was not deblended')
801 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag',
802 doc='The source is identified as a "pseudo" source and '
803 'was not deblended')
804 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
805 doc='Source had too many peaks; '
806 'only the brightest were included')
807 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
808 doc='Parent footprint covered too many pixels')
809 self.maskedKey = schema.addField('deblend_masked', type='Flag',
810 doc='Parent footprint had too many masked pixels')
811 # Convergence flags
812 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag',
813 doc='scarlet sed optimization did not converge before'
814 'config.maxIter')
815 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag',
816 doc='scarlet morph optimization did not converge before'
817 'config.maxIter')
818 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag',
819 type='Flag',
820 doc='at least one source in the blend'
821 'failed to converge')
822 # Error flags
823 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
824 doc="Deblending failed on source")
825 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25,
826 doc='Name of error if the blend failed')
827 # Deblended source fields
828 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center",
829 doc="Center used to apply constraints in scarlet",
830 unit="pixel")
831 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32,
832 doc="ID of the peak in the parent footprint. "
833 "This is not unique, but the combination of 'parent'"
834 "and 'peakId' should be for all child sources. "
835 "Top level blends with no parents have 'peakId=0'")
836 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count',
837 doc="The instFlux at the peak position of deblended mode")
838 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25,
839 doc="The type of model used, for example "
840 "MultiExtendedSource, SingleExtendedSource, PointSource")
841 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32,
842 doc="deblend_nPeaks from this records parent.")
843 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32,
844 doc="deblend_nChild from this records parent.")
845 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32,
846 doc="Flux measurement from scarlet")
847 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32,
848 doc="Final logL, used to identify regressions in scarlet.")
849 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag',
850 doc='Source had flux on the edge of the parent footprint')
851 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag',
852 doc="True when scarlet initializes sources "
853 "in the blend with a more accurate spectrum. "
854 "The algorithm uses a lot of memory, "
855 "so large dense blends will use "
856 "a less accurate initialization.")
857 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32,
858 doc="Number of components in a ScarletLiteSource. "
859 "If `config.version != 'lite'`then "
860 "this column is set to zero.")
861 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
862 doc='Deblender thought this source looked like a PSF')
863 # Blendedness/classification metrics
864 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32,
865 doc="Maximum overlap with all of the other neighbors flux "
866 "combined."
867 "This is useful as a metric for determining how blended a "
868 "source is because if it only overlaps with other sources "
869 "at or below the noise level, it is likely to be a mostly "
870 "isolated source in the deconvolved model frame.")
871 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32,
872 doc="This is the total flux from neighboring objects that "
873 "overlaps with this source.")
874 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32,
875 doc="This is the fraction of "
876 "`flux from neighbors/source flux` "
877 "for a given source within the source's"
878 "footprint.")
879 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32,
880 doc="The Bosch et al. 2018 metric for 'blendedness.' ")
882 @timeMethod
883 def run(self, mExposure, mergedSources):
884 """Get the psf from each exposure and then run deblend().
886 Parameters
887 ----------
888 mExposure : `MultibandExposure`
889 The exposures should be co-added images of the same
890 shape and region of the sky.
891 mergedSources : `SourceCatalog`
892 The merged `SourceCatalog` that contains parent footprints
893 to (potentially) deblend.
895 Returns
896 -------
897 templateCatalogs: dict
898 Keys are the names of the filters and the values are
899 `lsst.afw.table.source.source.SourceCatalog`'s.
900 These are catalogs with heavy footprints that are the templates
901 created by the multiband templates.
902 """
903 return self.deblend(mExposure, mergedSources)
905 @timeMethod
906 def deblend(self, mExposure, catalog):
907 """Deblend a data cube of multiband images
909 Parameters
910 ----------
911 mExposure : `MultibandExposure`
912 The exposures should be co-added images of the same
913 shape and region of the sky.
914 catalog : `SourceCatalog`
915 The merged `SourceCatalog` that contains parent footprints
916 to (potentially) deblend. The new deblended sources are
917 appended to this catalog in place.
919 Returns
920 -------
921 catalogs : `dict` or `None`
922 Keys are the names of the filters and the values are
923 `lsst.afw.table.source.source.SourceCatalog`'s.
924 These are catalogs with heavy footprints that are the templates
925 created by the multiband templates.
926 """
927 import time
929 # Cull footprints if required by ci
930 if self.config.useCiLimits:
931 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.",
932 len(catalog))
933 # Select parents with a number of children in the range
934 # config.ciDeblendChildRange
935 minChildren, maxChildren = self.config.ciDeblendChildRange
936 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog])
937 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
938 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
939 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range "
940 "indicated by ciDeblendChildRange. Adjust this range to include more "
941 "parents.")
942 # Keep all of the isolated parents and the first
943 # `ciNumParentsToDeblend` children
944 parents = nPeaks == 1
945 children = np.zeros((len(catalog),), dtype=bool)
946 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True
947 catalog = catalog[parents | children]
948 # We need to update the IdFactory, otherwise the the source ids
949 # will not be sequential
950 idFactory = catalog.getIdFactory()
951 maxId = np.max(catalog["id"])
952 idFactory.notify(maxId)
954 filters = mExposure.filters
955 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure))
956 periodicLog = PeriodicLogger(self.log)
958 # Create a set of wavelet coefficients if using wavelet initialization
959 if self.config.version == "lite" and self.config.morphImage == "wavelet":
960 images = mExposure.image.array
961 variance = mExposure.variance.array
962 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales)
963 else:
964 wavelets = None
966 # Add the NOT_DEBLENDED mask to the mask plane in each band
967 if self.config.notDeblendedMask:
968 for mask in mExposure.mask:
969 mask.addMaskPlane(self.config.notDeblendedMask)
971 # Initialize the persistable data model
972 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma)
973 dataModel = ScarletModelData(filters, modelPsf)
975 nParents = len(catalog)
976 nDeblendedParents = 0
977 skippedParents = []
978 for parentIndex in range(nParents):
979 parent = catalog[parentIndex]
980 foot = parent.getFootprint()
981 bbox = foot.getBBox()
982 peaks = foot.getPeaks()
984 # Since we use the first peak for the parent object, we should
985 # propagate its flags to the parent source.
986 parent.assign(peaks[0], self.peakSchemaMapper)
988 # Block of conditions for skipping a parent with multiple children
989 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None:
990 self._skipParent(parent, *skipArgs)
991 skippedParents.append(parentIndex)
992 continue
994 nDeblendedParents += 1
995 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks))
996 # Run the deblender
997 blendError = None
999 # Choose whether or not to use improved spectral initialization.
1000 # This significantly cuts down on the number of iterations
1001 # that the optimizer needs and usually results in a better
1002 # fit.
1003 # But using least squares on a very large blend causes memory
1004 # issues, so it is not done for large blends
1005 if self.config.setSpectra:
1006 if self.config.maxSpectrumCutoff <= 0:
1007 spectrumInit = True
1008 else:
1009 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff
1010 else:
1011 spectrumInit = False
1013 try:
1014 t0 = time.monotonic()
1015 # Build the parameter lists with the same ordering
1016 if self.config.version == "scarlet":
1017 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit)
1018 elif self.config.version == "lite":
1019 blend, skipped = deblend_lite(
1020 mExposure=mExposure,
1021 modelPsf=modelPsf,
1022 footprint=foot,
1023 config=self.config,
1024 spectrumInit=spectrumInit,
1025 wavelets=wavelets,
1026 )
1027 tf = time.monotonic()
1028 runtime = (tf-t0)*1000
1029 converged = _checkBlendConvergence(blend, self.config.relativeError)
1030 # Store the number of components in the blend
1031 if self.config.version == "lite":
1032 nComponents = len(blend.components)
1033 else:
1034 nComponents = 0
1035 nChild = len(blend.sources)
1036 # Catch all errors and filter out the ones that we know about
1037 except Exception as e:
1038 print("deblend failed")
1039 print(e)
1040 blendError = type(e).__name__
1041 if isinstance(e, ScarletGradientError):
1042 parent.set(self.iterKey, e.iterations)
1043 elif not isinstance(e, IncompleteDataError):
1044 blendError = "UnknownError"
1045 if self.config.catchFailures:
1046 # Make it easy to find UnknownErrors in the log file
1047 self.log.warn("UnknownError")
1048 import traceback
1049 traceback.print_exc()
1050 else:
1051 raise
1053 self._skipParent(
1054 parent=parent,
1055 skipKey=self.deblendFailedKey,
1056 logMessage=f"Unable to deblend source {parent.getId}: {blendError}",
1057 )
1058 parent.set(self.deblendErrorKey, blendError)
1059 skippedParents.append(parentIndex)
1060 continue
1062 # Update the parent record with the deblending results
1063 if self.config.version == "scarlet":
1064 logL = -blend.loss[-1] + blend.observations[0].log_norm
1065 elif self.config.version == "lite":
1066 logL = blend.loss[-1]
1067 self._updateParentRecord(
1068 parent=parent,
1069 nPeaks=len(peaks),
1070 nChild=nChild,
1071 nComponents=nComponents,
1072 runtime=runtime,
1073 iterations=len(blend.loss),
1074 logL=logL,
1075 spectrumInit=spectrumInit,
1076 converged=converged,
1077 )
1079 # Add each deblended source to the catalog
1080 for k, scarletSource in enumerate(blend.sources):
1081 # Skip any sources with no flux or that scarlet skipped because
1082 # it could not initialize
1083 if k in skipped or (self.config.version == "lite" and scarletSource.is_null):
1084 # No need to propagate anything
1085 continue
1086 parent.set(self.deblendSkippedKey, False)
1088 # Add all fields except the HeavyFootprint to the
1089 # source record
1090 sourceRecord = self._addChild(
1091 parent=parent,
1092 peak=scarletSource.detectedPeak,
1093 catalog=catalog,
1094 scarletSource=scarletSource,
1095 )
1096 scarletSource.recordId = sourceRecord.getId()
1097 scarletSource.peakId = scarletSource.detectedPeak.getId()
1099 # Store the blend information so that it can be persisted
1100 if self.config.version == "lite":
1101 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin())
1102 else:
1103 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin())
1104 dataModel.blends[parent.getId()] = blendData
1106 # Log a message if it has been a while since the last log.
1107 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents)
1109 # Clear the cached values in scarlet to clear out memory
1110 scarlet.cache.Cache._cache = {}
1112 # Update the mExposure mask with the footprint of skipped parents
1113 if self.config.notDeblendedMask:
1114 for mask in mExposure.mask:
1115 for parentIndex in skippedParents:
1116 fp = catalog[parentIndex].getFootprint()
1117 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
1119 self.log.info("Deblender results: of %d parent sources, %d were deblended, "
1120 "creating %d children, for a total of %d sources",
1121 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog))
1122 return catalog, dataModel
1124 def _isLargeFootprint(self, footprint):
1125 """Returns whether a Footprint is large
1127 'Large' is defined by thresholds on the area, size and axis ratio,
1128 and total area of the bounding box multiplied by
1129 the number of children.
1130 These may be disabled independently by configuring them to be
1131 non-positive.
1132 """
1133 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
1134 return True
1135 if self.config.maxFootprintSize > 0:
1136 bbox = footprint.getBBox()
1137 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
1138 return True
1139 if self.config.minFootprintAxisRatio > 0:
1140 axes = afwEll.Axes(footprint.getShape())
1141 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
1142 return True
1143 if self.config.maxAreaTimesPeaks > 0:
1144 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks:
1145 return True
1146 return False
1148 def _isMasked(self, footprint, mExposure):
1149 """Returns whether the footprint violates the mask limits
1151 Parameters
1152 ----------
1153 footprint : `lsst.afw.detection.Footprint`
1154 The footprint to check for masked pixels
1155 mMask : `lsst.afw.image.MaskX`
1156 The mask plane to check for masked pixels in the `footprint`.
1158 Returns
1159 -------
1160 isMasked : `bool`
1161 `True` if `self.config.maskPlaneLimits` is less than the
1162 fraction of pixels for a given mask in
1163 `self.config.maskLimits`.
1164 """
1165 bbox = footprint.getBBox()
1166 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0)
1167 size = float(footprint.getArea())
1168 for maskName, limit in self.config.maskLimits.items():
1169 maskVal = mExposure.mask.getPlaneBitMask(maskName)
1170 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin())
1171 # spanset of masked pixels
1172 maskedSpan = footprint.spans.intersect(_mask, maskVal)
1173 if (maskedSpan.getArea())/size > limit:
1174 return True
1175 return False
1177 def _skipParent(self, parent, skipKey, logMessage):
1178 """Update a parent record that is not being deblended.
1180 This is a fairly trivial function but is implemented to ensure
1181 that a skipped parent updates the appropriate columns
1182 consistently, and always has a flag to mark the reason that
1183 it is being skipped.
1185 Parameters
1186 ----------
1187 parent : `lsst.afw.table.source.source.SourceRecord`
1188 The parent record to flag as skipped.
1189 skipKey : `bool`
1190 The name of the flag to mark the reason for skipping.
1191 logMessage : `str`
1192 The message to display in a log.trace when a source
1193 is skipped.
1194 """
1195 if logMessage is not None:
1196 self.log.trace(logMessage)
1197 self._updateParentRecord(
1198 parent=parent,
1199 nPeaks=len(parent.getFootprint().peaks),
1200 nChild=0,
1201 nComponents=0,
1202 runtime=np.nan,
1203 iterations=0,
1204 logL=np.nan,
1205 spectrumInit=False,
1206 converged=False,
1207 )
1209 # Mark the source as skipped by the deblender and
1210 # flag the reason why.
1211 parent.set(self.deblendSkippedKey, True)
1212 parent.set(skipKey, True)
1214 def _checkSkipped(self, parent, mExposure):
1215 """Update a parent record that is not being deblended.
1217 This is a fairly trivial function but is implemented to ensure
1218 that a skipped parent updates the appropriate columns
1219 consistently, and always has a flag to mark the reason that
1220 it is being skipped.
1222 Parameters
1223 ----------
1224 parent : `lsst.afw.table.source.source.SourceRecord`
1225 The parent record to flag as skipped.
1226 mExposure : `MultibandExposure`
1227 The exposures should be co-added images of the same
1228 shape and region of the sky.
1229 Returns
1230 -------
1231 skip: `bool`
1232 `True` if the deblender will skip the parent
1233 """
1234 skipKey = None
1235 skipMessage = None
1236 footprint = parent.getFootprint()
1237 if len(footprint.peaks) < 2 and not self.config.processSingles:
1238 # Skip isolated sources unless processSingles is turned on.
1239 # Note: this does not flag isolated sources as skipped or
1240 # set the NOT_DEBLENDED mask in the exposure,
1241 # since these aren't really any skipped blends.
1242 skipKey = self.isolatedParentKey
1243 elif isPseudoSource(parent, self.config.pseudoColumns):
1244 # We also skip pseudo sources, like sky objects, which
1245 # are intended to be skipped.
1246 skipKey = self.pseudoKey
1247 if self._isLargeFootprint(footprint):
1248 # The footprint is above the maximum footprint size limit
1249 skipKey = self.tooBigKey
1250 skipMessage = f"Parent {parent.getId()}: skipping large footprint"
1251 elif self._isMasked(footprint, mExposure):
1252 # The footprint exceeds the maximum number of masked pixels
1253 skipKey = self.maskedKey
1254 skipMessage = f"Parent {parent.getId()}: skipping masked footprint"
1255 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks:
1256 # Unlike meas_deblender, in scarlet we skip the entire blend
1257 # if the number of peaks exceeds max peaks, since neglecting
1258 # to model any peaks often results in catastrophic failure
1259 # of scarlet to generate models for the brighter sources.
1260 skipKey = self.tooManyPeaksKey
1261 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks"
1262 if skipKey is not None:
1263 return (skipKey, skipMessage)
1264 return None
1266 def setSkipFlags(self, mExposure, catalog):
1267 """Set the skip flags for all of the parent sources
1269 This is mostly used for testing which parent sources will be deblended
1270 and which will be skipped based on the current configuration options.
1271 Skipped sources will have the appropriate flags set in place in the
1272 catalog.
1274 Parameters
1275 ----------
1276 mExposure : `MultibandExposure`
1277 The exposures should be co-added images of the same
1278 shape and region of the sky.
1279 catalog : `SourceCatalog`
1280 The merged `SourceCatalog` that contains parent footprints
1281 to (potentially) deblend. The new deblended sources are
1282 appended to this catalog in place.
1283 """
1284 for src in catalog:
1285 if skipArgs := self._checkSkipped(src, mExposure) is not None:
1286 self._skipParent(src, *skipArgs)
1288 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents,
1289 runtime, iterations, logL, spectrumInit, converged):
1290 """Update a parent record in all of the single band catalogs.
1292 Ensure that all locations that update a parent record,
1293 whether it is skipped or updated after deblending,
1294 update all of the appropriate columns.
1296 Parameters
1297 ----------
1298 parent : `lsst.afw.table.source.source.SourceRecord`
1299 The parent record to update.
1300 nPeaks : `int`
1301 Number of peaks in the parent footprint.
1302 nChild : `int`
1303 Number of children deblended from the parent.
1304 This may differ from `nPeaks` if some of the peaks
1305 were culled and have no deblended model.
1306 nComponents : `int`
1307 Total number of components in the parent.
1308 This is usually different than the number of children,
1309 since it is common for a single source to have multiple
1310 components.
1311 runtime : `float`
1312 Total runtime for deblending.
1313 iterations : `int`
1314 Total number of iterations in scarlet before convergence.
1315 logL : `float`
1316 Final log likelihood of the blend.
1317 spectrumInit : `bool`
1318 True when scarlet used `set_spectra` to initialize all
1319 sources with better initial intensities.
1320 converged : `bool`
1321 True when the optimizer reached convergence before
1322 reaching the maximum number of iterations.
1323 """
1324 parent.set(self.nPeaksKey, nPeaks)
1325 parent.set(self.nChildKey, nChild)
1326 parent.set(self.nComponentsKey, nComponents)
1327 parent.set(self.runtimeKey, runtime)
1328 parent.set(self.iterKey, iterations)
1329 parent.set(self.scarletLogLKey, logL)
1330 parent.set(self.scarletSpectrumInitKey, spectrumInit)
1331 parent.set(self.blendConvergenceFailedFlagKey, converged)
1333 def _addChild(self, parent, peak, catalog, scarletSource):
1334 """Add a child to a catalog.
1336 This creates a new child in the source catalog,
1337 assigning it a parent id, and adding all columns
1338 that are independent across all filter bands.
1340 Parameters
1341 ----------
1342 parent : `lsst.afw.table.source.source.SourceRecord`
1343 The parent of the new child record.
1344 peak : `lsst.afw.table.PeakRecord`
1345 The peak record for the peak from the parent peak catalog.
1346 catalog : `lsst.afw.table.source.source.SourceCatalog`
1347 The merged `SourceCatalog` that contains parent footprints
1348 to (potentially) deblend.
1349 scarletSource : `scarlet.Component`
1350 The scarlet model for the new source record.
1351 """
1352 src = catalog.addNew()
1353 for key in self.toCopyFromParent:
1354 src.set(key, parent.get(key))
1355 # The peak catalog is the same for all bands,
1356 # so we just use the first peak catalog
1357 src.assign(peak, self.peakSchemaMapper)
1358 src.setParent(parent.getId())
1359 src.set(self.nPeaksKey, 1)
1360 # Set the psf key based on whether or not the source was
1361 # deblended using the PointSource model.
1362 # This key is not that useful anymore since we now keep track of
1363 # `modelType`, but we continue to propagate it in case code downstream
1364 # is expecting it.
1365 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource")
1366 src.set(self.modelTypeKey, scarletSource.__class__.__name__)
1367 # We set the runtime to zero so that summing up the
1368 # runtime column will give the total time spent
1369 # running the deblender for the catalog.
1370 src.set(self.runtimeKey, 0)
1372 # Set the position of the peak from the parent footprint
1373 # This will make it easier to match the same source across
1374 # deblenders and across observations, where the peak
1375 # position is unlikely to change unless enough time passes
1376 # for a source to move on the sky.
1377 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"]))
1378 src.set(self.peakIdKey, peak["id"])
1380 # Store the number of components for the source
1381 src.set(self.nComponentsKey, len(scarletSource.components))
1383 # Propagate columns from the parent to the child
1384 for parentColumn, childColumn in self.config.columnInheritance.items():
1385 src.set(childColumn, parent.get(parentColumn))
1387 return src