Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%
448 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-05 19:34 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-05 19:34 +0000
1# This file is part of meas_extensions_scarlet.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from dataclasses import dataclass
23from functools import partial
24import logging
25import numpy as np
26import scarlet
27from scarlet.psf import ImagePSF, GaussianPSF
28from scarlet import Blend, Frame, Observation
29from scarlet.renderer import ConvolutionRenderer
30from scarlet.detect import get_detect_wavelets
31from scarlet.initialization import init_all_sources
32from scarlet import lite
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36from lsst.geom import Point2I, Point2D
37import lsst.afw.geom.ellipses as afwEll
38import lsst.afw.image as afwImage
39import lsst.afw.detection as afwDet
40import lsst.afw.table as afwTable
41from lsst.utils.logging import PeriodicLogger
42from lsst.utils.timer import timeMethod
44from .source import bboxToScarletBox
45from .io import ScarletModelData, scarletToData, scarletLiteToData
47# Scarlet and proxmin have a different definition of log levels than the stack,
48# so even "warnings" occur far more often than we would like.
49# So for now we only display scarlet and proxmin errors, as all other
50# scarlet outputs would be considered "TRACE" by our standards.
51scarletLogger = logging.getLogger("scarlet")
52scarletLogger.setLevel(logging.ERROR)
53proxminLogger = logging.getLogger("proxmin")
54proxminLogger.setLevel(logging.ERROR)
56__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"]
58logger = logging.getLogger(__name__)
61class IncompleteDataError(Exception):
62 """The PSF could not be computed due to incomplete data
63 """
64 pass
67class ScarletGradientError(Exception):
68 """An error occurred during optimization
70 This error occurs when the optimizer encounters
71 a NaN value while calculating the gradient.
72 """
73 def __init__(self, iterations, sources):
74 self.iterations = iterations
75 self.sources = sources
76 msg = ("ScalarGradientError in iteration {0}. "
77 "NaN values introduced in sources {1}")
78 self.message = msg.format(iterations, sources)
80 def __str__(self):
81 return self.message
84def _checkBlendConvergence(blend, f_rel):
85 """Check whether or not a blend has converged
86 """
87 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1])
88 convergence = f_rel * np.abs(blend.loss[-1])
89 return deltaLoss < convergence
92def isPseudoSource(source, pseudoColumns):
93 """Check if a source is a pseudo source.
95 This is mostly for skipping sky objects,
96 but any other column can also be added to disable
97 deblending on a parent or individual source when
98 set to `True`.
100 Parameters
101 ----------
102 source : `lsst.afw.table.source.source.SourceRecord`
103 The source to check for the pseudo bit.
104 pseudoColumns : `list` of `str`
105 A list of columns to check for pseudo sources.
106 """
107 isPseudo = False
108 for col in pseudoColumns:
109 try:
110 isPseudo |= source[col]
111 except KeyError:
112 pass
113 return isPseudo
116def deblend(mExposure, footprint, config, spectrumInit):
117 """Deblend a parent footprint
119 Parameters
120 ----------
121 mExposure : `lsst.image.MultibandExposure`
122 The multiband exposure containing the image,
123 mask, and variance data.
124 footprint : `lsst.detection.Footprint`
125 The footprint of the parent to deblend.
126 config : `ScarletDeblendConfig`
127 Configuration of the deblending task.
128 spectrumInit : `bool`
129 Whether or not to initialize the spectrum.
131 Returns
132 -------
133 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData`
134 The persistable representation of a `scarlet.Blend`.
135 skipped : `list` of `int`
136 The indices of any children that failed to initialize
137 and were skipped.
138 spectrumInit : `bool`
139 Whether or not all of the sources were initialized by jointly
140 fitting their SED's. This provides a better initialization
141 but created memory issues when a blend is too large or
142 contains too many sources.
143 """
144 # Extract coordinates from each MultiColorPeak
145 bbox = footprint.getBBox()
147 # Create the data array from the masked images
148 images = mExposure.image[:, bbox].array
150 # Use the inverse variance as the weights
151 if config.useWeights:
152 weights = 1/mExposure.variance[:, bbox].array
153 else:
154 weights = np.ones_like(images)
155 badPixels = mExposure.mask.getPlaneBitMask(config.badMask)
156 mask = mExposure.mask[:, bbox].array & badPixels
157 weights[mask > 0] = 0
159 # Mask out the pixels outside the footprint
160 weights *= footprint.spans.asArray()
162 psfCenter = footprint.getCentroid()
163 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32)
164 psfs = ImagePSF(psfs)
165 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters))
167 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters)
168 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters)
169 if config.convolutionType == "fft":
170 observation.match(frame)
171 elif config.convolutionType == "real":
172 renderer = ConvolutionRenderer(observation, frame, convolution_type="real")
173 observation.match(frame, renderer=renderer)
174 else:
175 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType))
177 assert(config.sourceModel in ["single", "double", "compact", "fit"])
179 # Set the appropriate number of components
180 if config.sourceModel == "single":
181 maxComponents = 1
182 elif config.sourceModel == "double":
183 maxComponents = 2
184 elif config.sourceModel == "compact":
185 maxComponents = 0
186 elif config.sourceModel == "point":
187 raise NotImplementedError("Point source photometry is currently not implemented")
188 elif config.sourceModel == "fit":
189 # It is likely in the future that there will be some heuristic
190 # used to determine what type of model to use for each source,
191 # but that has not yet been implemented (see DM-22551)
192 raise NotImplementedError("sourceModel 'fit' has not been implemented yet")
194 # Convert the centers to pixel coordinates
195 xmin = bbox.getMinX()
196 ymin = bbox.getMinY()
197 centers = [
198 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
199 for peak in footprint.peaks
200 if not isPseudoSource(peak, config.pseudoColumns)
201 ]
203 # Only deblend sources that can be initialized
204 sources, skipped = init_all_sources(
205 frame=frame,
206 centers=centers,
207 observations=observation,
208 thresh=config.morphThresh,
209 max_components=maxComponents,
210 min_snr=config.minSNR,
211 shifting=False,
212 fallback=config.fallback,
213 silent=config.catchFailures,
214 set_spectra=spectrumInit,
215 )
217 # Attach the peak to all of the initialized sources
218 srcIndex = 0
219 for k, center in enumerate(centers):
220 if k not in skipped:
221 # This is just to make sure that there isn't a coding bug
222 assert np.all(sources[srcIndex].center == center)
223 # Store the record for the peak with the appropriate source
224 sources[srcIndex].detectedPeak = footprint.peaks[k]
225 srcIndex += 1
227 # Create the blend and attempt to optimize it
228 blend = Blend(sources, observation)
229 try:
230 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError)
231 except ArithmeticError:
232 # This occurs when a gradient update produces a NaN value
233 # This is usually due to a source initialized with a
234 # negative SED or no flux, often because the peak
235 # is a noise fluctuation in one band and not a real source.
236 iterations = len(blend.loss)
237 failedSources = []
238 for k, src in enumerate(sources):
239 if np.any(~np.isfinite(src.get_model())):
240 failedSources.append(k)
241 raise ScarletGradientError(iterations, failedSources)
243 # Store the location of the PSF center for storage
244 blend.psfCenter = (psfCenter.x, psfCenter.y)
246 return blend, skipped
249def buildLiteObservation(
250 modelPsf,
251 psfCenter,
252 mExposure,
253 footprint=None,
254 badPixelMasks=None,
255 useWeights=True,
256 convolutionType="real",
257):
258 """Generate a LiteObservation from a set of parameters.
260 Make the generation and reconstruction of a scarlet model consistent
261 by building a `LiteObservation` from a set of parameters.
263 Parameters
264 ----------
265 modelPsf : `numpy.ndarray`
266 The 2D model of the PSF in the partially deconvolved space.
267 psfCenter : `tuple` or `Point2I` or `Point2D`
268 The location `(x, y)` used as the center of the PSF.
269 mExposure : `lsst.afw.image.multiband.MultibandExposure`
270 The multi-band exposure that the model represents.
271 If `mExposure` is `None` then no image, variance, or weights are
272 attached to the observation.
273 footprint : `lsst.afw.detection.Footprint`
274 The footprint that is being fit.
275 If `Footprint` is `None` then the weights are not updated to mask
276 out pixels not contained in the footprint.
277 badPixelMasks : `list` of `str`
278 The keys from the bit mask plane used to mask out pixels
279 during the fit.
280 If `badPixelMasks` is `None` then the default values from
281 `ScarletDeblendConfig.badMask` is used.
282 useWeights : `bool`
283 Whether or not fitting should use inverse variance weights to
284 calculate the log-likelihood.
285 convolutionType : `str`
286 The type of convolution to use (either "real" or "fft").
287 When reconstructing an image it is advised to use "real" to avoid
288 polluting the footprint with
290 Returns
291 -------
292 observation : `scarlet.lite.LiteObservation`
293 The observation constructed from the input parameters.
294 """
295 # Initialize the observed PSFs
296 if not isinstance(psfCenter, Point2D):
297 psfCenter = Point2D(*psfCenter)
298 psfModels = mExposure.computePsfKernelImage(psfCenter)
300 # Use the inverse variance as the weights
301 if useWeights:
302 weights = 1/mExposure.variance.array
303 else:
304 # Mask out bad pixels
305 weights = np.ones_like(mExposure.image.array)
306 if badPixelMasks is None:
307 badPixelMasks = ScarletDeblendConfig().badMask
308 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks)
309 mask = mExposure.mask.array & badPixels
310 weights[mask > 0] = 0
312 if footprint is not None:
313 # Mask out the pixels outside the footprint
314 weights *= footprint.spans.asArray()
316 return lite.LiteObservation(
317 images=mExposure.image.array,
318 variance=mExposure.variance.array,
319 weights=weights,
320 psfs=psfModels,
321 model_psf=modelPsf[None, :, :],
322 convolution_mode=convolutionType,
323 )
326def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None):
327 """Deblend a parent footprint
329 Parameters
330 ----------
331 mExposure : `lsst.image.MultibandExposure`
332 - The multiband exposure containing the image,
333 mask, and variance data
334 footprint : `lsst.detection.Footprint`
335 - The footprint of the parent to deblend
336 config : `ScarletDeblendConfig`
337 - Configuration of the deblending task
338 """
339 # Extract coordinates from each MultiColorPeak
340 bbox = footprint.getBBox()
341 psfCenter = footprint.getCentroid()
343 observation = buildLiteObservation(
344 modelPsf=modelPsf,
345 psfCenter=psfCenter,
346 mExposure=mExposure[:, bbox],
347 footprint=footprint,
348 badPixelMasks=config.badMask,
349 useWeights=config.useWeights,
350 convolutionType=config.convolutionType,
351 )
353 # Convert the centers to pixel coordinates
354 xmin = bbox.getMinX()
355 ymin = bbox.getMinY()
356 centers = [
357 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
358 for peak in footprint.peaks
359 if not isPseudoSource(peak, config.pseudoColumns)
360 ]
362 # Initialize the sources
363 if config.morphImage == "chi2":
364 sources = lite.init_all_sources_main(
365 observation,
366 centers,
367 min_snr=config.minSNR,
368 thresh=config.morphThresh,
369 )
370 elif config.morphImage == "wavelet":
371 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin())
372 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)]
373 sources = lite.init_all_sources_wavelets(
374 observation,
375 centers,
376 use_psf=False,
377 wavelets=_wavelets,
378 min_snr=config.minSNR,
379 )
380 else:
381 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.")
383 # Set the optimizer
384 if config.optimizer == "adaprox":
385 parameterization = partial(
386 lite.init_adaprox_component,
387 bg_thresh=config.backgroundThresh,
388 max_prox_iter=config.maxProxIter,
389 )
390 elif config.optimizer == "fista":
391 parameterization = partial(
392 lite.init_fista_component,
393 bg_thresh=config.backgroundThresh,
394 )
395 else:
396 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.")
397 sources = lite.parameterize_sources(sources, observation, parameterization)
399 # Attach the peak to all of the initialized sources
400 for k, center in enumerate(centers):
401 # This is just to make sure that there isn't a coding bug
402 if len(sources[k].components) > 0 and np.any(sources[k].center != center):
403 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}")
404 # Store the record for the peak with the appropriate source
405 sources[k].detectedPeak = footprint.peaks[k]
407 blend = lite.LiteBlend(sources, observation)
409 # Initialize each source with its best fit spectrum
410 if spectrumInit:
411 blend.fit_spectra()
413 # Set the sources that could not be initialized and were skipped
414 skipped = [src for src in sources if src.is_null]
416 blend.fit(
417 max_iter=config.maxIter,
418 e_rel=config.relativeError,
419 min_iter=config.minIter,
420 reweight=False,
421 )
423 # Store the location of the PSF center for storage
424 blend.psfCenter = (psfCenter.x, psfCenter.y)
426 return blend, skipped
429@dataclass
430class DeblenderMetrics:
431 """Metrics and measurements made on single sources.
433 Store deblender metrics to be added as attributes to a scarlet source
434 before it is converted into a `SourceRecord`.
435 When DM-34414 is finished this class will be eliminated and the metrics
436 will be added to the schema using a pipeline task that calculates them
437 from the stored deconvolved models.
439 All of the parameters are one dimensional numpy arrays,
440 with an element for each band in the observed images.
442 `maxOverlap` is useful as a metric for determining how blended a source
443 is because if it only overlaps with other sources at or below
444 the noise level, it is likely to be a mostly isolated source
445 in the deconvolved model frame.
447 `fluxOverlapFraction` is potentially more useful than the canonical
448 "blendedness" (or purity) metric because it accounts for potential
449 biases created during deblending by not weighting the overlapping
450 flux with the flux of this sources model.
452 Attributes
453 ----------
454 maxOverlap : `numpy.ndarray`
455 The maximum overlap that the source has with its neighbors in
456 a single pixel.
457 fluxOverlap : `numpy.ndarray`
458 The total flux from neighbors overlapping with the current source.
459 fluxOverlapFraction : `numpy.ndarray`
460 The fraction of `flux from neighbors/source flux` for a
461 given source within the source's footprint.
462 blendedness : `numpy.ndarray`
463 The metric for determining how blended a source is using the
464 Bosch et al. 2018 metric for "blendedness." Note that some
465 surveys use the term "purity," which is `1-blendedness`.
466 """
467 maxOverlap: np.array
468 fluxOverlap: np.array
469 fluxOverlapFraction: np.array
470 blendedness: np.array
473def setDeblenderMetrics(blend):
474 """Set metrics that can be used to evalute the deblender accuracy
476 This function calculates the `DeblenderMetrics` for each source in the
477 blend, and assigns it to that sources `metrics` property in place.
479 Parameters
480 ----------
481 blend : `scarlet.lite.Blend`
482 The blend containing the sources to measure.
483 """
484 # Store the full model of the scene for comparison
485 blendModel = blend.get_model()
486 for k, src in enumerate(blend.sources):
487 # Extract the source model in the full bounding box
488 model = src.get_model(bbox=blend.bbox)
489 # The footprint is the 2D array of non-zero pixels in each band
490 footprint = np.bitwise_or.reduce(model > 0, axis=0)
491 # Calculate the metrics.
492 # See `DeblenderMetrics` for a description of each metric.
493 neighborOverlap = (blendModel-model) * footprint[None, :, :]
494 maxOverlap = np.max(neighborOverlap, axis=(1, 2))
495 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2))
496 fluxModel = np.sum(model, axis=(1, 2))
497 fluxOverlapFraction = np.zeros((len(model), ), dtype=float)
498 isFinite = fluxModel > 0
499 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite]
500 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2))
501 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness)
504class ScarletDeblendConfig(pexConfig.Config):
505 """MultibandDeblendConfig
507 Configuration for the multiband deblender.
508 The parameters are organized by the parameter types, which are
509 - Stopping Criteria: Used to determine if the fit has converged
510 - Position Fitting Criteria: Used to fit the positions of the peaks
511 - Constraints: Used to apply constraints to the peaks and their components
512 - Other: Parameters that don't fit into the above categories
513 """
514 # Stopping Criteria
515 minIter = pexConfig.Field(dtype=int, default=1,
516 doc="Minimum number of iterations before the optimizer is allowed to stop.")
517 maxIter = pexConfig.Field(dtype=int, default=300,
518 doc=("Maximum number of iterations to deblend a single parent"))
519 relativeError = pexConfig.Field(dtype=float, default=1e-2,
520 doc=("Change in the loss function between iterations to exit fitter. "
521 "Typically this is `1e-2` if measurements will be made on the "
522 "flux re-distributed models and `1e-4` when making measurements "
523 "on the models themselves."))
525 # Constraints
526 morphThresh = pexConfig.Field(dtype=float, default=1,
527 doc="Fraction of background RMS a pixel must have"
528 "to be included in the initial morphology")
529 # Lite Parameters
530 # All of these parameters (except version) are only valid if version='lite'
531 version = pexConfig.ChoiceField(
532 dtype=str,
533 default="lite",
534 allowed={
535 "scarlet": "main scarlet version (likely to be deprecated soon)",
536 "lite": "Optimized version of scarlet for survey data from a single instrument",
537 },
538 doc="The version of scarlet to use.",
539 )
540 optimizer = pexConfig.ChoiceField(
541 dtype=str,
542 default="adaprox",
543 allowed={
544 "adaprox": "Proximal ADAM optimization",
545 "fista": "Accelerated proximal gradient method",
546 },
547 doc="The optimizer to use for fitting parameters and is only used when version='lite'",
548 )
549 morphImage = pexConfig.ChoiceField(
550 dtype=str,
551 default="chi2",
552 allowed={
553 "chi2": "Initialize sources on a chi^2 image made from all available bands",
554 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image",
555 },
556 doc="The type of image to use for initializing the morphology. "
557 "Must be either 'chi2' or 'wavelet'. "
558 )
559 backgroundThresh = pexConfig.Field(
560 dtype=float,
561 default=0.25,
562 doc="Fraction of background to use for a sparsity threshold. "
563 "This prevents sources from growing unrealistically outside "
564 "the parent footprint while still modeling flux correctly "
565 "for bright sources."
566 )
567 maxProxIter = pexConfig.Field(
568 dtype=int,
569 default=1,
570 doc="Maximum number of proximal operator iterations inside of each "
571 "iteration of the optimizer. "
572 "This config field is only used if version='lite' and optimizer='adaprox'."
573 )
574 waveletScales = pexConfig.Field(
575 dtype=int,
576 default=5,
577 doc="Number of wavelet scales to use for wavelet initialization. "
578 "This field is only used when `version`='lite' and `morphImage`='wavelet'."
579 )
581 # Other scarlet paremeters
582 useWeights = pexConfig.Field(
583 dtype=bool, default=True,
584 doc=("Whether or not use use inverse variance weighting."
585 "If `useWeights` is `False` then flat weights are used"))
586 modelPsfSize = pexConfig.Field(
587 dtype=int, default=11,
588 doc="Model PSF side length in pixels")
589 modelPsfSigma = pexConfig.Field(
590 dtype=float, default=0.8,
591 doc="Define sigma for the model frame PSF")
592 minSNR = pexConfig.Field(
593 dtype=float, default=50,
594 doc="Minimum Signal to noise to accept the source."
595 "Sources with lower flux will be initialized with the PSF but updated "
596 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).")
597 saveTemplates = pexConfig.Field(
598 dtype=bool, default=True,
599 doc="Whether or not to save the SEDs and templates")
600 processSingles = pexConfig.Field(
601 dtype=bool, default=True,
602 doc="Whether or not to process isolated sources in the deblender")
603 convolutionType = pexConfig.Field(
604 dtype=str, default="fft",
605 doc="Type of convolution to render the model to the observations.\n"
606 "- 'fft': perform convolutions in Fourier space\n"
607 "- 'real': peform convolutions in real space.")
608 sourceModel = pexConfig.Field(
609 dtype=str, default="double",
610 doc=("How to determine which model to use for sources, from\n"
611 "- 'single': use a single component for all sources\n"
612 "- 'double': use a bulge disk model for all sources\n"
613 "- 'compact': use a single component model, initialzed with a point source morphology, "
614 " for all sources\n"
615 "- 'point': use a point-source model for all sources\n"
616 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"),
617 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.",
618 )
619 setSpectra = pexConfig.Field(
620 dtype=bool, default=True,
621 doc="Whether or not to solve for the best-fit spectra during initialization. "
622 "This makes initialization slightly longer, as it requires a convolution "
623 "to set the optimal spectra, but results in a much better initial log-likelihood "
624 "and reduced total runtime, with convergence in fewer iterations."
625 "This option is only used when "
626 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.")
628 # Mask-plane restrictions
629 badMask = pexConfig.ListField(
630 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"],
631 doc="Whether or not to process isolated sources in the deblender")
632 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
633 doc="Mask planes to ignore when performing statistics")
634 maskLimits = pexConfig.DictField(
635 keytype=str,
636 itemtype=float,
637 default={},
638 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
639 "Sources violating this limit will not be deblended. "
640 "If the fraction is `0` then the limit is a single pixel."),
641 )
643 # Size restrictions
644 maxNumberOfPeaks = pexConfig.Field(
645 dtype=int, default=200,
646 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
647 " (<= 0: unlimited)"))
648 maxFootprintArea = pexConfig.Field(
649 dtype=int, default=100_000,
650 doc=("Maximum area for footprints before they are ignored as large; "
651 "non-positive means no threshold applied"))
652 maxAreaTimesPeaks = pexConfig.Field(
653 dtype=int, default=10_000_000,
654 doc=("Maximum rectangular footprint area * nPeaks in the footprint. "
655 "This was introduced in DM-33690 to prevent fields that are crowded or have a "
656 "LSB galaxy that causes memory intensive initialization in scarlet from dominating "
657 "the overall runtime and/or causing the task to run out of memory. "
658 "(<= 0: unlimited)")
659 )
660 maxFootprintSize = pexConfig.Field(
661 dtype=int, default=0,
662 doc=("Maximum linear dimension for footprints before they are ignored "
663 "as large; non-positive means no threshold applied"))
664 minFootprintAxisRatio = pexConfig.Field(
665 dtype=float, default=0.0,
666 doc=("Minimum axis ratio for footprints before they are ignored "
667 "as large; non-positive means no threshold applied"))
668 maxSpectrumCutoff = pexConfig.Field(
669 dtype=int, default=1_000_000,
670 doc=("Maximum number of pixels * number of sources in a blend. "
671 "This is different than `maxFootprintArea` because this isn't "
672 "the footprint area but the area of the bounding box that "
673 "contains the footprint, and is also multiplied by the number of"
674 "sources in the footprint. This prevents large skinny blends with "
675 "a high density of sources from running out of memory. "
676 "If `maxSpectrumCutoff == -1` then there is no cutoff.")
677 )
678 # Failure modes
679 fallback = pexConfig.Field(
680 dtype=bool, default=True,
681 doc="Whether or not to fallback to a smaller number of components if a source does not initialize"
682 )
683 notDeblendedMask = pexConfig.Field(
684 dtype=str, default="NOT_DEBLENDED", optional=True,
685 doc="Mask name for footprints not deblended, or None")
686 catchFailures = pexConfig.Field(
687 dtype=bool, default=True,
688 doc=("If True, catch exceptions thrown by the deblender, log them, "
689 "and set a flag on the parent, instead of letting them propagate up"))
691 # Other options
692 columnInheritance = pexConfig.DictField(
693 keytype=str, itemtype=str, default={
694 "deblend_nChild": "deblend_parentNChild",
695 "deblend_nPeaks": "deblend_parentNPeaks",
696 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag",
697 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag",
698 },
699 doc="Columns to pass from the parent to the child. "
700 "The key is the name of the column for the parent record, "
701 "the value is the name of the column to use for the child."
702 )
703 pseudoColumns = pexConfig.ListField(
704 dtype=str, default=['merge_peak_sky', 'sky_source'],
705 doc="Names of flags which should never be deblended."
706 )
708 # Logging option(s)
709 loggingInterval = pexConfig.Field(
710 dtype=int, default=600,
711 doc="Interval (in seconds) to log messages (at VERBOSE level) while deblending sources.",
712 deprecated="This field is no longer used and will be removed in v25.",
713 )
714 # Testing options
715 # Some obs packages and ci packages run the full pipeline on a small
716 # subset of data to test that the pipeline is functioning properly.
717 # This is not meant as scientific validation, so it can be useful
718 # to only run on a small subset of the data that is large enough to
719 # test the desired pipeline features but not so long that the deblender
720 # is the tall pole in terms of execution times.
721 useCiLimits = pexConfig.Field(
722 dtype=bool, default=False,
723 doc="Limit the number of sources deblended for CI to prevent long build times")
724 ciDeblendChildRange = pexConfig.ListField(
725 dtype=int, default=[5, 10],
726 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
727 "If `useCiLimits==False` then this parameter is ignored.")
728 ciNumParentsToDeblend = pexConfig.Field(
729 dtype=int, default=10,
730 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
731 "within `ciDebledChildRange`. "
732 "If `useCiLimits==False` then this parameter is ignored.")
735class ScarletDeblendTask(pipeBase.Task):
736 """ScarletDeblendTask
738 Split blended sources into individual sources.
740 This task has no return value; it only modifies the SourceCatalog in-place.
741 """
742 ConfigClass = ScarletDeblendConfig
743 _DefaultName = "scarletDeblend"
745 def __init__(self, schema, peakSchema=None, **kwargs):
746 """Create the task, adding necessary fields to the given schema.
748 Parameters
749 ----------
750 schema : `lsst.afw.table.schema.schema.Schema`
751 Schema object for measurement fields; will be modified in-place.
752 peakSchema : `lsst.afw.table.schema.schema.Schema`
753 Schema of Footprint Peaks that will be passed to the deblender.
754 Any fields beyond the PeakTable minimal schema will be transferred
755 to the main source Schema. If None, no fields will be transferred
756 from the Peaks.
757 filters : list of str
758 Names of the filters used for the eposures. This is needed to store
759 the SED as a field
760 **kwargs
761 Passed to Task.__init__.
762 """
763 pipeBase.Task.__init__(self, **kwargs)
765 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
766 if peakSchema is None:
767 # In this case, the peakSchemaMapper will transfer nothing, but
768 # we'll still have one
769 # to simplify downstream code
770 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
771 else:
772 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
773 for item in peakSchema:
774 if item.key not in peakMinimalSchema:
775 self.peakSchemaMapper.addMapping(item.key, item.field)
776 # Because SchemaMapper makes a copy of the output schema
777 # you give its ctor, it isn't updating this Schema in
778 # place. That's probably a design flaw, but in the
779 # meantime, we'll keep that schema in sync with the
780 # peakSchemaMapper.getOutputSchema() manually, by adding
781 # the same fields to both.
782 schema.addField(item.field)
783 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
784 self._addSchemaKeys(schema)
785 self.schema = schema
786 self.toCopyFromParent = [item.key for item in self.schema
787 if item.field.getName().startswith("merge_footprint")]
789 def _addSchemaKeys(self, schema):
790 """Add deblender specific keys to the schema
791 """
792 # Parent (blend) fields
793 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms')
794 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge')
795 self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
796 doc='Number of children this object has (defaults to 0)')
797 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32,
798 doc="Number of initial peaks in the blend. "
799 "This includes peaks that may have been culled "
800 "during deblending or failed to deblend")
801 # Skipped flags
802 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
803 doc="Deblender skipped this source")
804 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag',
805 doc='The source has only a single peak '
806 'and was not deblended')
807 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag',
808 doc='The source is identified as a "pseudo" source and '
809 'was not deblended')
810 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
811 doc='Source had too many peaks; '
812 'only the brightest were included')
813 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
814 doc='Parent footprint covered too many pixels')
815 self.maskedKey = schema.addField('deblend_masked', type='Flag',
816 doc='Parent footprint had too many masked pixels')
817 # Convergence flags
818 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag',
819 doc='scarlet sed optimization did not converge before'
820 'config.maxIter')
821 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag',
822 doc='scarlet morph optimization did not converge before'
823 'config.maxIter')
824 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag',
825 type='Flag',
826 doc='at least one source in the blend'
827 'failed to converge')
828 # Error flags
829 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
830 doc="Deblending failed on source")
831 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25,
832 doc='Name of error if the blend failed')
833 # Deblended source fields
834 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center",
835 doc="Center used to apply constraints in scarlet",
836 unit="pixel")
837 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32,
838 doc="ID of the peak in the parent footprint. "
839 "This is not unique, but the combination of 'parent'"
840 "and 'peakId' should be for all child sources. "
841 "Top level blends with no parents have 'peakId=0'")
842 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count',
843 doc="The instFlux at the peak position of deblended mode")
844 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25,
845 doc="The type of model used, for example "
846 "MultiExtendedSource, SingleExtendedSource, PointSource")
847 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32,
848 doc="deblend_nPeaks from this records parent.")
849 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32,
850 doc="deblend_nChild from this records parent.")
851 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32,
852 doc="Flux measurement from scarlet")
853 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32,
854 doc="Final logL, used to identify regressions in scarlet.")
855 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag',
856 doc='Source had flux on the edge of the parent footprint')
857 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag',
858 doc="True when scarlet initializes sources "
859 "in the blend with a more accurate spectrum. "
860 "The algorithm uses a lot of memory, "
861 "so large dense blends will use "
862 "a less accurate initialization.")
863 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32,
864 doc="Number of components in a ScarletLiteSource. "
865 "If `config.version != 'lite'`then "
866 "this column is set to zero.")
867 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
868 doc='Deblender thought this source looked like a PSF')
869 # Blendedness/classification metrics
870 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32,
871 doc="Maximum overlap with all of the other neighbors flux "
872 "combined."
873 "This is useful as a metric for determining how blended a "
874 "source is because if it only overlaps with other sources "
875 "at or below the noise level, it is likely to be a mostly "
876 "isolated source in the deconvolved model frame.")
877 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32,
878 doc="This is the total flux from neighboring objects that "
879 "overlaps with this source.")
880 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32,
881 doc="This is the fraction of "
882 "`flux from neighbors/source flux` "
883 "for a given source within the source's"
884 "footprint.")
885 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32,
886 doc="The Bosch et al. 2018 metric for 'blendedness.' ")
888 @timeMethod
889 def run(self, mExposure, mergedSources):
890 """Get the psf from each exposure and then run deblend().
892 Parameters
893 ----------
894 mExposure : `MultibandExposure`
895 The exposures should be co-added images of the same
896 shape and region of the sky.
897 mergedSources : `SourceCatalog`
898 The merged `SourceCatalog` that contains parent footprints
899 to (potentially) deblend.
901 Returns
902 -------
903 templateCatalogs: dict
904 Keys are the names of the filters and the values are
905 `lsst.afw.table.source.source.SourceCatalog`'s.
906 These are catalogs with heavy footprints that are the templates
907 created by the multiband templates.
908 """
909 return self.deblend(mExposure, mergedSources)
911 @timeMethod
912 def deblend(self, mExposure, catalog):
913 """Deblend a data cube of multiband images
915 Parameters
916 ----------
917 mExposure : `MultibandExposure`
918 The exposures should be co-added images of the same
919 shape and region of the sky.
920 catalog : `SourceCatalog`
921 The merged `SourceCatalog` that contains parent footprints
922 to (potentially) deblend. The new deblended sources are
923 appended to this catalog in place.
925 Returns
926 -------
927 catalogs : `dict` or `None`
928 Keys are the names of the filters and the values are
929 `lsst.afw.table.source.source.SourceCatalog`'s.
930 These are catalogs with heavy footprints that are the templates
931 created by the multiband templates.
932 """
933 import time
935 # Cull footprints if required by ci
936 if self.config.useCiLimits:
937 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.",
938 len(catalog))
939 # Select parents with a number of children in the range
940 # config.ciDeblendChildRange
941 minChildren, maxChildren = self.config.ciDeblendChildRange
942 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog])
943 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
944 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
945 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range "
946 "indicated by ciDeblendChildRange. Adjust this range to include more "
947 "parents.")
948 # Keep all of the isolated parents and the first
949 # `ciNumParentsToDeblend` children
950 parents = nPeaks == 1
951 children = np.zeros((len(catalog),), dtype=bool)
952 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True
953 catalog = catalog[parents | children]
954 # We need to update the IdFactory, otherwise the the source ids
955 # will not be sequential
956 idFactory = catalog.getIdFactory()
957 maxId = np.max(catalog["id"])
958 idFactory.notify(maxId)
960 filters = mExposure.filters
961 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure))
962 periodicLog = PeriodicLogger(self.log)
964 # Create a set of wavelet coefficients if using wavelet initialization
965 if self.config.version == "lite" and self.config.morphImage == "wavelet":
966 images = mExposure.image.array
967 variance = mExposure.variance.array
968 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales)
969 else:
970 wavelets = None
972 # Add the NOT_DEBLENDED mask to the mask plane in each band
973 if self.config.notDeblendedMask:
974 for mask in mExposure.mask:
975 mask.addMaskPlane(self.config.notDeblendedMask)
977 # Initialize the persistable data model
978 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma)
979 dataModel = ScarletModelData(filters, modelPsf)
981 nParents = len(catalog)
982 nDeblendedParents = 0
983 skippedParents = []
984 for parentIndex in range(nParents):
985 parent = catalog[parentIndex]
986 foot = parent.getFootprint()
987 bbox = foot.getBBox()
988 peaks = foot.getPeaks()
990 # Since we use the first peak for the parent object, we should
991 # propagate its flags to the parent source.
992 parent.assign(peaks[0], self.peakSchemaMapper)
994 # Block of conditions for skipping a parent with multiple children
995 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None:
996 self._skipParent(parent, *skipArgs)
997 skippedParents.append(parentIndex)
998 continue
1000 nDeblendedParents += 1
1001 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks))
1002 # Run the deblender
1003 blendError = None
1005 # Choose whether or not to use improved spectral initialization.
1006 # This significantly cuts down on the number of iterations
1007 # that the optimizer needs and usually results in a better
1008 # fit.
1009 # But using least squares on a very large blend causes memory
1010 # issues, so it is not done for large blends
1011 if self.config.setSpectra:
1012 if self.config.maxSpectrumCutoff <= 0:
1013 spectrumInit = True
1014 else:
1015 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff
1016 else:
1017 spectrumInit = False
1019 try:
1020 t0 = time.monotonic()
1021 # Build the parameter lists with the same ordering
1022 if self.config.version == "scarlet":
1023 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit)
1024 elif self.config.version == "lite":
1025 blend, skipped = deblend_lite(
1026 mExposure=mExposure,
1027 modelPsf=modelPsf,
1028 footprint=foot,
1029 config=self.config,
1030 spectrumInit=spectrumInit,
1031 wavelets=wavelets,
1032 )
1033 tf = time.monotonic()
1034 runtime = (tf-t0)*1000
1035 converged = _checkBlendConvergence(blend, self.config.relativeError)
1036 # Store the number of components in the blend
1037 if self.config.version == "lite":
1038 nComponents = len(blend.components)
1039 else:
1040 nComponents = 0
1041 nChild = len(blend.sources)
1042 # Catch all errors and filter out the ones that we know about
1043 except Exception as e:
1044 print("deblend failed")
1045 print(e)
1046 blendError = type(e).__name__
1047 if isinstance(e, ScarletGradientError):
1048 parent.set(self.iterKey, e.iterations)
1049 elif not isinstance(e, IncompleteDataError):
1050 blendError = "UnknownError"
1051 if self.config.catchFailures:
1052 # Make it easy to find UnknownErrors in the log file
1053 self.log.warn("UnknownError")
1054 import traceback
1055 traceback.print_exc()
1056 else:
1057 raise
1059 self._skipParent(
1060 parent=parent,
1061 skipKey=self.deblendFailedKey,
1062 logMessage=f"Unable to deblend source {parent.getId}: {blendError}",
1063 )
1064 parent.set(self.deblendErrorKey, blendError)
1065 skippedParents.append(parentIndex)
1066 continue
1068 # Update the parent record with the deblending results
1069 if self.config.version == "scarlet":
1070 logL = -blend.loss[-1] + blend.observations[0].log_norm
1071 elif self.config.version == "lite":
1072 logL = blend.loss[-1]
1073 self._updateParentRecord(
1074 parent=parent,
1075 nPeaks=len(peaks),
1076 nChild=nChild,
1077 nComponents=nComponents,
1078 runtime=runtime,
1079 iterations=len(blend.loss),
1080 logL=logL,
1081 spectrumInit=spectrumInit,
1082 converged=converged,
1083 )
1085 # Add each deblended source to the catalog
1086 for k, scarletSource in enumerate(blend.sources):
1087 # Skip any sources with no flux or that scarlet skipped because
1088 # it could not initialize
1089 if k in skipped or (self.config.version == "lite" and scarletSource.is_null):
1090 # No need to propagate anything
1091 continue
1092 parent.set(self.deblendSkippedKey, False)
1094 # Add all fields except the HeavyFootprint to the
1095 # source record
1096 sourceRecord = self._addChild(
1097 parent=parent,
1098 peak=scarletSource.detectedPeak,
1099 catalog=catalog,
1100 scarletSource=scarletSource,
1101 )
1102 scarletSource.recordId = sourceRecord.getId()
1103 scarletSource.peakId = scarletSource.detectedPeak.getId()
1105 # Store the blend information so that it can be persisted
1106 if self.config.version == "lite":
1107 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin())
1108 else:
1109 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin())
1110 dataModel.blends[parent.getId()] = blendData
1112 # Log a message if it has been a while since the last log.
1113 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents)
1115 # Clear the cached values in scarlet to clear out memory
1116 scarlet.cache.Cache._cache = {}
1118 # Update the mExposure mask with the footprint of skipped parents
1119 if self.config.notDeblendedMask:
1120 for mask in mExposure.mask:
1121 for parentIndex in skippedParents:
1122 fp = catalog[parentIndex].getFootprint()
1123 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
1125 self.log.info("Deblender results: of %d parent sources, %d were deblended, "
1126 "creating %d children, for a total of %d sources",
1127 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog))
1128 return catalog, dataModel
1130 def _isLargeFootprint(self, footprint):
1131 """Returns whether a Footprint is large
1133 'Large' is defined by thresholds on the area, size and axis ratio,
1134 and total area of the bounding box multiplied by
1135 the number of children.
1136 These may be disabled independently by configuring them to be
1137 non-positive.
1138 """
1139 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
1140 return True
1141 if self.config.maxFootprintSize > 0:
1142 bbox = footprint.getBBox()
1143 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
1144 return True
1145 if self.config.minFootprintAxisRatio > 0:
1146 axes = afwEll.Axes(footprint.getShape())
1147 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
1148 return True
1149 if self.config.maxAreaTimesPeaks > 0:
1150 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks:
1151 return True
1152 return False
1154 def _isMasked(self, footprint, mExposure):
1155 """Returns whether the footprint violates the mask limits
1157 Parameters
1158 ----------
1159 footprint : `lsst.afw.detection.Footprint`
1160 The footprint to check for masked pixels
1161 mMask : `lsst.afw.image.MaskX`
1162 The mask plane to check for masked pixels in the `footprint`.
1164 Returns
1165 -------
1166 isMasked : `bool`
1167 `True` if `self.config.maskPlaneLimits` is less than the
1168 fraction of pixels for a given mask in
1169 `self.config.maskLimits`.
1170 """
1171 bbox = footprint.getBBox()
1172 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0)
1173 size = float(footprint.getArea())
1174 for maskName, limit in self.config.maskLimits.items():
1175 maskVal = mExposure.mask.getPlaneBitMask(maskName)
1176 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin())
1177 # spanset of masked pixels
1178 maskedSpan = footprint.spans.intersect(_mask, maskVal)
1179 if (maskedSpan.getArea())/size > limit:
1180 return True
1181 return False
1183 def _skipParent(self, parent, skipKey, logMessage):
1184 """Update a parent record that is not being deblended.
1186 This is a fairly trivial function but is implemented to ensure
1187 that a skipped parent updates the appropriate columns
1188 consistently, and always has a flag to mark the reason that
1189 it is being skipped.
1191 Parameters
1192 ----------
1193 parent : `lsst.afw.table.source.source.SourceRecord`
1194 The parent record to flag as skipped.
1195 skipKey : `bool`
1196 The name of the flag to mark the reason for skipping.
1197 logMessage : `str`
1198 The message to display in a log.trace when a source
1199 is skipped.
1200 """
1201 if logMessage is not None:
1202 self.log.trace(logMessage)
1203 self._updateParentRecord(
1204 parent=parent,
1205 nPeaks=len(parent.getFootprint().peaks),
1206 nChild=0,
1207 nComponents=0,
1208 runtime=np.nan,
1209 iterations=0,
1210 logL=np.nan,
1211 spectrumInit=False,
1212 converged=False,
1213 )
1215 # Mark the source as skipped by the deblender and
1216 # flag the reason why.
1217 parent.set(self.deblendSkippedKey, True)
1218 parent.set(skipKey, True)
1220 def _checkSkipped(self, parent, mExposure):
1221 """Update a parent record that is not being deblended.
1223 This is a fairly trivial function but is implemented to ensure
1224 that a skipped parent updates the appropriate columns
1225 consistently, and always has a flag to mark the reason that
1226 it is being skipped.
1228 Parameters
1229 ----------
1230 parent : `lsst.afw.table.source.source.SourceRecord`
1231 The parent record to flag as skipped.
1232 mExposure : `MultibandExposure`
1233 The exposures should be co-added images of the same
1234 shape and region of the sky.
1235 Returns
1236 -------
1237 skip: `bool`
1238 `True` if the deblender will skip the parent
1239 """
1240 skipKey = None
1241 skipMessage = None
1242 footprint = parent.getFootprint()
1243 if len(footprint.peaks) < 2 and not self.config.processSingles:
1244 # Skip isolated sources unless processSingles is turned on.
1245 # Note: this does not flag isolated sources as skipped or
1246 # set the NOT_DEBLENDED mask in the exposure,
1247 # since these aren't really any skipped blends.
1248 skipKey = self.isolatedParentKey
1249 elif isPseudoSource(parent, self.config.pseudoColumns):
1250 # We also skip pseudo sources, like sky objects, which
1251 # are intended to be skipped.
1252 skipKey = self.pseudoKey
1253 if self._isLargeFootprint(footprint):
1254 # The footprint is above the maximum footprint size limit
1255 skipKey = self.tooBigKey
1256 skipMessage = f"Parent {parent.getId()}: skipping large footprint"
1257 elif self._isMasked(footprint, mExposure):
1258 # The footprint exceeds the maximum number of masked pixels
1259 skipKey = self.maskedKey
1260 skipMessage = f"Parent {parent.getId()}: skipping masked footprint"
1261 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks:
1262 # Unlike meas_deblender, in scarlet we skip the entire blend
1263 # if the number of peaks exceeds max peaks, since neglecting
1264 # to model any peaks often results in catastrophic failure
1265 # of scarlet to generate models for the brighter sources.
1266 skipKey = self.tooManyPeaksKey
1267 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks"
1268 if skipKey is not None:
1269 return (skipKey, skipMessage)
1270 return None
1272 def setSkipFlags(self, mExposure, catalog):
1273 """Set the skip flags for all of the parent sources
1275 This is mostly used for testing which parent sources will be deblended
1276 and which will be skipped based on the current configuration options.
1277 Skipped sources will have the appropriate flags set in place in the
1278 catalog.
1280 Parameters
1281 ----------
1282 mExposure : `MultibandExposure`
1283 The exposures should be co-added images of the same
1284 shape and region of the sky.
1285 catalog : `SourceCatalog`
1286 The merged `SourceCatalog` that contains parent footprints
1287 to (potentially) deblend. The new deblended sources are
1288 appended to this catalog in place.
1289 """
1290 for src in catalog:
1291 if skipArgs := self._checkSkipped(src, mExposure) is not None:
1292 self._skipParent(src, *skipArgs)
1294 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents,
1295 runtime, iterations, logL, spectrumInit, converged):
1296 """Update a parent record in all of the single band catalogs.
1298 Ensure that all locations that update a parent record,
1299 whether it is skipped or updated after deblending,
1300 update all of the appropriate columns.
1302 Parameters
1303 ----------
1304 parent : `lsst.afw.table.source.source.SourceRecord`
1305 The parent record to update.
1306 nPeaks : `int`
1307 Number of peaks in the parent footprint.
1308 nChild : `int`
1309 Number of children deblended from the parent.
1310 This may differ from `nPeaks` if some of the peaks
1311 were culled and have no deblended model.
1312 nComponents : `int`
1313 Total number of components in the parent.
1314 This is usually different than the number of children,
1315 since it is common for a single source to have multiple
1316 components.
1317 runtime : `float`
1318 Total runtime for deblending.
1319 iterations : `int`
1320 Total number of iterations in scarlet before convergence.
1321 logL : `float`
1322 Final log likelihood of the blend.
1323 spectrumInit : `bool`
1324 True when scarlet used `set_spectra` to initialize all
1325 sources with better initial intensities.
1326 converged : `bool`
1327 True when the optimizer reached convergence before
1328 reaching the maximum number of iterations.
1329 """
1330 parent.set(self.nPeaksKey, nPeaks)
1331 parent.set(self.nChildKey, nChild)
1332 parent.set(self.nComponentsKey, nComponents)
1333 parent.set(self.runtimeKey, runtime)
1334 parent.set(self.iterKey, iterations)
1335 parent.set(self.scarletLogLKey, logL)
1336 parent.set(self.scarletSpectrumInitKey, spectrumInit)
1337 parent.set(self.blendConvergenceFailedFlagKey, converged)
1339 def _addChild(self, parent, peak, catalog, scarletSource):
1340 """Add a child to a catalog.
1342 This creates a new child in the source catalog,
1343 assigning it a parent id, and adding all columns
1344 that are independent across all filter bands.
1346 Parameters
1347 ----------
1348 parent : `lsst.afw.table.source.source.SourceRecord`
1349 The parent of the new child record.
1350 peak : `lsst.afw.table.PeakRecord`
1351 The peak record for the peak from the parent peak catalog.
1352 catalog : `lsst.afw.table.source.source.SourceCatalog`
1353 The merged `SourceCatalog` that contains parent footprints
1354 to (potentially) deblend.
1355 scarletSource : `scarlet.Component`
1356 The scarlet model for the new source record.
1357 """
1358 src = catalog.addNew()
1359 for key in self.toCopyFromParent:
1360 src.set(key, parent.get(key))
1361 # The peak catalog is the same for all bands,
1362 # so we just use the first peak catalog
1363 src.assign(peak, self.peakSchemaMapper)
1364 src.setParent(parent.getId())
1365 src.set(self.nPeaksKey, 1)
1366 # Set the psf key based on whether or not the source was
1367 # deblended using the PointSource model.
1368 # This key is not that useful anymore since we now keep track of
1369 # `modelType`, but we continue to propagate it in case code downstream
1370 # is expecting it.
1371 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource")
1372 src.set(self.modelTypeKey, scarletSource.__class__.__name__)
1373 # We set the runtime to zero so that summing up the
1374 # runtime column will give the total time spent
1375 # running the deblender for the catalog.
1376 src.set(self.runtimeKey, 0)
1378 # Set the position of the peak from the parent footprint
1379 # This will make it easier to match the same source across
1380 # deblenders and across observations, where the peak
1381 # position is unlikely to change unless enough time passes
1382 # for a source to move on the sky.
1383 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"]))
1384 src.set(self.peakIdKey, peak["id"])
1386 # Store the number of components for the source
1387 src.set(self.nComponentsKey, len(scarletSource.components))
1389 # Propagate columns from the parent to the child
1390 for parentColumn, childColumn in self.config.columnInheritance.items():
1391 src.set(childColumn, parent.get(parentColumn))
1393 return src