Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%
470 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-02 10:09 +0000
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-02 10:09 +0000
1# This file is part of meas_extensions_scarlet.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from dataclasses import dataclass
23from functools import partial
24import logging
25import numpy as np
26import scarlet
27from scarlet.psf import ImagePSF, GaussianPSF
28from scarlet import Blend, Frame, Observation
29from scarlet.renderer import ConvolutionRenderer
30from scarlet.detect import get_detect_wavelets
31from scarlet.initialization import init_all_sources
32from scarlet import lite
34import lsst.pex.config as pexConfig
35from lsst.pex.exceptions import InvalidParameterError
36import lsst.pipe.base as pipeBase
37from lsst.geom import Point2I, Box2I, Point2D
38import lsst.afw.geom.ellipses as afwEll
39import lsst.afw.image as afwImage
40import lsst.afw.detection as afwDet
41import lsst.afw.table as afwTable
42from lsst.utils.logging import PeriodicLogger
43from lsst.utils.timer import timeMethod
45from .source import bboxToScarletBox
46from .io import ScarletModelData, scarletToData, scarletLiteToData
48# Scarlet and proxmin have a different definition of log levels than the stack,
49# so even "warnings" occur far more often than we would like.
50# So for now we only display scarlet and proxmin errors, as all other
51# scarlet outputs would be considered "TRACE" by our standards.
52scarletLogger = logging.getLogger("scarlet")
53scarletLogger.setLevel(logging.ERROR)
54proxminLogger = logging.getLogger("proxmin")
55proxminLogger.setLevel(logging.ERROR)
57__all__ = ["deblend", "ScarletDeblendConfig", "ScarletDeblendTask"]
59logger = logging.getLogger(__name__)
62class IncompleteDataError(Exception):
63 """The PSF could not be computed due to incomplete data
64 """
65 pass
68class ScarletGradientError(Exception):
69 """An error occurred during optimization
71 This error occurs when the optimizer encounters
72 a NaN value while calculating the gradient.
73 """
74 def __init__(self, iterations, sources):
75 self.iterations = iterations
76 self.sources = sources
77 msg = ("ScalarGradientError in iteration {0}. "
78 "NaN values introduced in sources {1}")
79 self.message = msg.format(iterations, sources)
81 def __str__(self):
82 return self.message
85def _checkBlendConvergence(blend, f_rel):
86 """Check whether or not a blend has converged
87 """
88 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1])
89 convergence = f_rel * np.abs(blend.loss[-1])
90 return deltaLoss < convergence
93def _computePsfImage(psfModels, position, filters):
94 """Get a multiband PSF image
96 The PSF Kernel Image is computed for each band
97 and combined into a (filter, y, x) array.
99 Parameters
100 ----------
101 psfList : `list` of `lsst.afw.detection.Psf`
102 The list of PSFs in each band.
103 position : `Point2D` or `tuple`
104 Coordinates to evaluate the PSF.
105 Returns
106 -------
107 psfs: `np.ndarray`
108 The multiband PSF image.
109 """
110 psfs = []
111 # Make the coordinates into a Point2D (if necessary)
112 if not isinstance(position, Point2D):
113 position = Point2D(position[0], position[1])
115 for bidx, psfModel in enumerate(psfModels):
116 try:
117 psf = psfModel.computeKernelImage(position)
118 psfs.append(psf)
119 except InvalidParameterError:
120 # This band failed to compute the PSF due to incomplete data
121 # at that location. This is unlikely to be a problem for Rubin,
122 # however the edges of some HSC COSMOS fields contain incomplete
123 # data in some bands, so we track this error to distinguish it
124 # from unknown errors.
125 msg = "Failed to compute PSF at {} in band {}"
126 raise IncompleteDataError(msg.format(position, filters[bidx])) from None
128 left = np.min([psf.getBBox().getMinX() for psf in psfs])
129 bottom = np.min([psf.getBBox().getMinY() for psf in psfs])
130 right = np.max([psf.getBBox().getMaxX() for psf in psfs])
131 top = np.max([psf.getBBox().getMaxY() for psf in psfs])
132 bbox = Box2I(Point2I(left, bottom), Point2I(right, top))
133 psfs = np.array([afwImage.utils.projectImage(psf, bbox).array for psf in psfs])
134 return psfs
137def getFootprintMask(footprint, mExposure):
138 """Mask pixels outside the footprint
140 Parameters
141 ----------
142 mExposure : `lsst.image.MultibandExposure`
143 - The multiband exposure containing the image,
144 mask, and variance data
145 footprint : `lsst.detection.Footprint`
146 - The footprint of the parent to deblend
148 Returns
149 -------
150 footprintMask : array
151 Boolean array with pixels not in the footprint set to one.
152 """
153 bbox = footprint.getBBox()
154 fpMask = afwImage.Mask(bbox)
155 footprint.spans.setMask(fpMask, 1)
156 fpMask = ~fpMask.getArray().astype(bool)
157 return fpMask
160def isPseudoSource(source, pseudoColumns):
161 """Check if a source is a pseudo source.
163 This is mostly for skipping sky objects,
164 but any other column can also be added to disable
165 deblending on a parent or individual source when
166 set to `True`.
168 Parameters
169 ----------
170 source : `lsst.afw.table.source.source.SourceRecord`
171 The source to check for the pseudo bit.
172 pseudoColumns : `list` of `str`
173 A list of columns to check for pseudo sources.
174 """
175 isPseudo = False
176 for col in pseudoColumns:
177 try:
178 isPseudo |= source[col]
179 except KeyError:
180 pass
181 return isPseudo
184def deblend(mExposure, footprint, config, spectrumInit):
185 """Deblend a parent footprint
187 Parameters
188 ----------
189 mExposure : `lsst.image.MultibandExposure`
190 The multiband exposure containing the image,
191 mask, and variance data.
192 footprint : `lsst.detection.Footprint`
193 The footprint of the parent to deblend.
194 config : `ScarletDeblendConfig`
195 Configuration of the deblending task.
196 spectrumInit : `bool`
197 Whether or not to initialize the spectrum.
199 Returns
200 -------
201 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData`
202 The persistable representation of a `scarlet.Blend`.
203 skipped : `list` of `int`
204 The indices of any children that failed to initialize
205 and were skipped.
206 spectrumInit : `bool`
207 Whether or not all of the sources were initialized by jointly
208 fitting their SED's. This provides a better initialization
209 but created memory issues when a blend is too large or
210 contains too many sources.
211 """
212 # Extract coordinates from each MultiColorPeak
213 bbox = footprint.getBBox()
215 # Create the data array from the masked images
216 images = mExposure.image[:, bbox].array
218 # Use the inverse variance as the weights
219 if config.useWeights:
220 weights = 1/mExposure.variance[:, bbox].array
221 else:
222 weights = np.ones_like(images)
223 badPixels = mExposure.mask.getPlaneBitMask(config.badMask)
224 mask = mExposure.mask[:, bbox].array & badPixels
225 weights[mask > 0] = 0
227 # Mask out the pixels outside the footprint
228 mask = getFootprintMask(footprint, mExposure)
229 weights *= ~mask
231 psfCenter = footprint.getCentroid()
232 psfModels = [exp.getPsf() for exp in mExposure]
233 psfs = _computePsfImage(psfModels, psfCenter, mExposure.filters).astype(np.float32)
234 psfs = ImagePSF(psfs)
235 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters))
237 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters)
238 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters)
239 if config.convolutionType == "fft":
240 observation.match(frame)
241 elif config.convolutionType == "real":
242 renderer = ConvolutionRenderer(observation, frame, convolution_type="real")
243 observation.match(frame, renderer=renderer)
244 else:
245 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType))
247 assert(config.sourceModel in ["single", "double", "compact", "fit"])
249 # Set the appropriate number of components
250 if config.sourceModel == "single":
251 maxComponents = 1
252 elif config.sourceModel == "double":
253 maxComponents = 2
254 elif config.sourceModel == "compact":
255 maxComponents = 0
256 elif config.sourceModel == "point":
257 raise NotImplementedError("Point source photometry is currently not implemented")
258 elif config.sourceModel == "fit":
259 # It is likely in the future that there will be some heuristic
260 # used to determine what type of model to use for each source,
261 # but that has not yet been implemented (see DM-22551)
262 raise NotImplementedError("sourceModel 'fit' has not been implemented yet")
264 # Convert the centers to pixel coordinates
265 xmin = bbox.getMinX()
266 ymin = bbox.getMinY()
267 centers = [
268 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
269 for peak in footprint.peaks
270 if not isPseudoSource(peak, config.pseudoColumns)
271 ]
273 # Only deblend sources that can be initialized
274 sources, skipped = init_all_sources(
275 frame=frame,
276 centers=centers,
277 observations=observation,
278 thresh=config.morphThresh,
279 max_components=maxComponents,
280 min_snr=config.minSNR,
281 shifting=False,
282 fallback=config.fallback,
283 silent=config.catchFailures,
284 set_spectra=spectrumInit,
285 )
287 # Attach the peak to all of the initialized sources
288 srcIndex = 0
289 for k, center in enumerate(centers):
290 if k not in skipped:
291 # This is just to make sure that there isn't a coding bug
292 assert np.all(sources[srcIndex].center == center)
293 # Store the record for the peak with the appropriate source
294 sources[srcIndex].detectedPeak = footprint.peaks[k]
295 srcIndex += 1
297 # Create the blend and attempt to optimize it
298 blend = Blend(sources, observation)
299 try:
300 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError)
301 except ArithmeticError:
302 # This occurs when a gradient update produces a NaN value
303 # This is usually due to a source initialized with a
304 # negative SED or no flux, often because the peak
305 # is a noise fluctuation in one band and not a real source.
306 iterations = len(blend.loss)
307 failedSources = []
308 for k, src in enumerate(sources):
309 if np.any(~np.isfinite(src.get_model())):
310 failedSources.append(k)
311 raise ScarletGradientError(iterations, failedSources)
313 # Store the location of the PSF center for storage
314 blend.psfCenter = (psfCenter.x, psfCenter.y)
316 return blend, skipped
319def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None):
320 """Deblend a parent footprint
322 Parameters
323 ----------
324 mExposure : `lsst.image.MultibandExposure`
325 - The multiband exposure containing the image,
326 mask, and variance data
327 footprint : `lsst.detection.Footprint`
328 - The footprint of the parent to deblend
329 config : `ScarletDeblendConfig`
330 - Configuration of the deblending task
331 """
332 # Extract coordinates from each MultiColorPeak
333 bbox = footprint.getBBox()
335 # Create the data array from the masked images
336 images = mExposure.image[:, bbox].array
337 variance = mExposure.variance[:, bbox].array
339 # Use the inverse variance as the weights
340 if config.useWeights:
341 weights = 1/mExposure.variance[:, bbox].array
342 else:
343 weights = np.ones_like(images)
344 badPixels = mExposure.mask.getPlaneBitMask(config.badMask)
345 mask = mExposure.mask[:, bbox].array & badPixels
346 weights[mask > 0] = 0
348 # Mask out the pixels outside the footprint
349 mask = getFootprintMask(footprint, mExposure)
350 weights *= ~mask
352 psfCenter = footprint.getCentroid()
353 psfModels = [exp.getPsf() for exp in mExposure]
354 psfs = _computePsfImage(psfModels, psfCenter, mExposure.filters).astype(np.float32)
356 observation = lite.LiteObservation(
357 images=images,
358 variance=variance,
359 weights=weights,
360 psfs=psfs,
361 model_psf=modelPsf[None, :, :],
362 convolution_mode=config.convolutionType,
363 )
365 # Convert the centers to pixel coordinates
366 xmin = bbox.getMinX()
367 ymin = bbox.getMinY()
368 centers = [
369 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
370 for peak in footprint.peaks
371 if not isPseudoSource(peak, config.pseudoColumns)
372 ]
374 # Initialize the sources
375 if config.morphImage == "chi2":
376 sources = lite.init_all_sources_main(
377 observation,
378 centers,
379 min_snr=config.minSNR,
380 thresh=config.morphThresh,
381 )
382 elif config.morphImage == "wavelet":
383 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin())
384 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)]
385 sources = lite.init_all_sources_wavelets(
386 observation,
387 centers,
388 use_psf=False,
389 wavelets=_wavelets,
390 min_snr=config.minSNR,
391 )
392 else:
393 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.")
395 # Set the optimizer
396 if config.optimizer == "adaprox":
397 parameterization = partial(
398 lite.init_adaprox_component,
399 bg_thresh=config.backgroundThresh,
400 max_prox_iter=config.maxProxIter,
401 )
402 elif config.optimizer == "fista":
403 parameterization = partial(
404 lite.init_fista_component,
405 bg_thresh=config.backgroundThresh,
406 )
407 else:
408 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.")
409 sources = lite.parameterize_sources(sources, observation, parameterization)
411 # Attach the peak to all of the initialized sources
412 for k, center in enumerate(centers):
413 # This is just to make sure that there isn't a coding bug
414 if len(sources[k].components) > 0 and np.any(sources[k].center != center):
415 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}")
416 # Store the record for the peak with the appropriate source
417 sources[k].detectedPeak = footprint.peaks[k]
419 blend = lite.LiteBlend(sources, observation)
421 # Initialize each source with its best fit spectrum
422 if spectrumInit:
423 blend.fit_spectra()
425 # Set the sources that could not be initialized and were skipped
426 skipped = [src for src in sources if src.is_null]
428 blend.fit(
429 max_iter=config.maxIter,
430 e_rel=config.relativeError,
431 min_iter=config.minIter,
432 reweight=False,
433 )
435 # Store the location of the PSF center for storage
436 blend.psfCenter = (psfCenter.x, psfCenter.y)
438 return blend, skipped
441@dataclass
442class DeblenderMetrics:
443 """Metrics and measurements made on single sources.
445 Store deblender metrics to be added as attributes to a scarlet source
446 before it is converted into a `SourceRecord`.
447 When DM-34414 is finished this class will be eliminated and the metrics
448 will be added to the schema using a pipeline task that calculates them
449 from the stored deconvolved models.
451 All of the parameters are one dimensional numpy arrays,
452 with an element for each band in the observed images.
454 `maxOverlap` is useful as a metric for determining how blended a source
455 is because if it only overlaps with other sources at or below
456 the noise level, it is likely to be a mostly isolated source
457 in the deconvolved model frame.
459 `fluxOverlapFraction` is potentially more useful than the canonical
460 "blendedness" (or purity) metric because it accounts for potential
461 biases created during deblending by not weighting the overlapping
462 flux with the flux of this sources model.
464 Attributes
465 ----------
466 maxOverlap : `numpy.ndarray`
467 The maximum overlap that the source has with its neighbors in
468 a single pixel.
469 fluxOverlap : `numpy.ndarray`
470 The total flux from neighbors overlapping with the current source.
471 fluxOverlapFraction : `numpy.ndarray`
472 The fraction of `flux from neighbors/source flux` for a
473 given source within the source's footprint.
474 blendedness : `numpy.ndarray`
475 The metric for determining how blended a source is using the
476 Bosch et al. 2018 metric for "blendedness." Note that some
477 surveys use the term "purity," which is `1-blendedness`.
478 """
479 maxOverlap: np.array
480 fluxOverlap: np.array
481 fluxOverlapFraction: np.array
482 blendedness: np.array
485def setDeblenderMetrics(blend):
486 """Set metrics that can be used to evalute the deblender accuracy
488 This function calculates the `DeblenderMetrics` for each source in the
489 blend, and assigns it to that sources `metrics` property in place.
491 Parameters
492 ----------
493 blend : `scarlet.lite.Blend`
494 The blend containing the sources to measure.
495 """
496 # Store the full model of the scene for comparison
497 blendModel = blend.get_model()
498 for k, src in enumerate(blend.sources):
499 # Extract the source model in the full bounding box
500 model = src.get_model(bbox=blend.bbox)
501 # The footprint is the 2D array of non-zero pixels in each band
502 footprint = np.bitwise_or.reduce(model > 0, axis=0)
503 # Calculate the metrics.
504 # See `DeblenderMetrics` for a description of each metric.
505 neighborOverlap = (blendModel-model) * footprint[None, :, :]
506 maxOverlap = np.max(neighborOverlap, axis=(1, 2))
507 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2))
508 fluxModel = np.sum(model, axis=(1, 2))
509 fluxOverlapFraction = np.zeros((len(model), ), dtype=float)
510 isFinite = fluxModel > 0
511 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite]
512 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2))
513 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness)
516class ScarletDeblendConfig(pexConfig.Config):
517 """MultibandDeblendConfig
519 Configuration for the multiband deblender.
520 The parameters are organized by the parameter types, which are
521 - Stopping Criteria: Used to determine if the fit has converged
522 - Position Fitting Criteria: Used to fit the positions of the peaks
523 - Constraints: Used to apply constraints to the peaks and their components
524 - Other: Parameters that don't fit into the above categories
525 """
526 # Stopping Criteria
527 minIter = pexConfig.Field(dtype=int, default=1,
528 doc="Minimum number of iterations before the optimizer is allowed to stop.")
529 maxIter = pexConfig.Field(dtype=int, default=300,
530 doc=("Maximum number of iterations to deblend a single parent"))
531 relativeError = pexConfig.Field(dtype=float, default=1e-2,
532 doc=("Change in the loss function between iterations to exit fitter. "
533 "Typically this is `1e-2` if measurements will be made on the "
534 "flux re-distributed models and `1e-4` when making measurements "
535 "on the models themselves."))
537 # Constraints
538 morphThresh = pexConfig.Field(dtype=float, default=1,
539 doc="Fraction of background RMS a pixel must have"
540 "to be included in the initial morphology")
541 # Lite Parameters
542 # All of these parameters (except version) are only valid if version='lite'
543 version = pexConfig.ChoiceField(
544 dtype=str,
545 default="lite",
546 allowed={
547 "scarlet": "main scarlet version (likely to be deprecated soon)",
548 "lite": "Optimized version of scarlet for survey data from a single instrument",
549 },
550 doc="The version of scarlet to use.",
551 )
552 optimizer = pexConfig.ChoiceField(
553 dtype=str,
554 default="adaprox",
555 allowed={
556 "adaprox": "Proximal ADAM optimization",
557 "fista": "Accelerated proximal gradient method",
558 },
559 doc="The optimizer to use for fitting parameters and is only used when version='lite'",
560 )
561 morphImage = pexConfig.ChoiceField(
562 dtype=str,
563 default="chi2",
564 allowed={
565 "chi2": "Initialize sources on a chi^2 image made from all available bands",
566 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image",
567 },
568 doc="The type of image to use for initializing the morphology. "
569 "Must be either 'chi2' or 'wavelet'. "
570 )
571 backgroundThresh = pexConfig.Field(
572 dtype=float,
573 default=0.25,
574 doc="Fraction of background to use for a sparsity threshold. "
575 "This prevents sources from growing unrealistically outside "
576 "the parent footprint while still modeling flux correctly "
577 "for bright sources."
578 )
579 maxProxIter = pexConfig.Field(
580 dtype=int,
581 default=1,
582 doc="Maximum number of proximal operator iterations inside of each "
583 "iteration of the optimizer. "
584 "This config field is only used if version='lite' and optimizer='adaprox'."
585 )
586 waveletScales = pexConfig.Field(
587 dtype=int,
588 default=5,
589 doc="Number of wavelet scales to use for wavelet initialization. "
590 "This field is only used when `version`='lite' and `morphImage`='wavelet'."
591 )
593 # Other scarlet paremeters
594 useWeights = pexConfig.Field(
595 dtype=bool, default=True,
596 doc=("Whether or not use use inverse variance weighting."
597 "If `useWeights` is `False` then flat weights are used"))
598 modelPsfSize = pexConfig.Field(
599 dtype=int, default=11,
600 doc="Model PSF side length in pixels")
601 modelPsfSigma = pexConfig.Field(
602 dtype=float, default=0.8,
603 doc="Define sigma for the model frame PSF")
604 minSNR = pexConfig.Field(
605 dtype=float, default=50,
606 doc="Minimum Signal to noise to accept the source."
607 "Sources with lower flux will be initialized with the PSF but updated "
608 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).")
609 saveTemplates = pexConfig.Field(
610 dtype=bool, default=True,
611 doc="Whether or not to save the SEDs and templates")
612 processSingles = pexConfig.Field(
613 dtype=bool, default=True,
614 doc="Whether or not to process isolated sources in the deblender")
615 convolutionType = pexConfig.Field(
616 dtype=str, default="fft",
617 doc="Type of convolution to render the model to the observations.\n"
618 "- 'fft': perform convolutions in Fourier space\n"
619 "- 'real': peform convolutions in real space.")
620 sourceModel = pexConfig.Field(
621 dtype=str, default="double",
622 doc=("How to determine which model to use for sources, from\n"
623 "- 'single': use a single component for all sources\n"
624 "- 'double': use a bulge disk model for all sources\n"
625 "- 'compact': use a single component model, initialzed with a point source morphology, "
626 " for all sources\n"
627 "- 'point': use a point-source model for all sources\n"
628 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"),
629 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.",
630 )
631 setSpectra = pexConfig.Field(
632 dtype=bool, default=True,
633 doc="Whether or not to solve for the best-fit spectra during initialization. "
634 "This makes initialization slightly longer, as it requires a convolution "
635 "to set the optimal spectra, but results in a much better initial log-likelihood "
636 "and reduced total runtime, with convergence in fewer iterations."
637 "This option is only used when "
638 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.")
640 # Mask-plane restrictions
641 badMask = pexConfig.ListField(
642 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"],
643 doc="Whether or not to process isolated sources in the deblender")
644 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
645 doc="Mask planes to ignore when performing statistics")
646 maskLimits = pexConfig.DictField(
647 keytype=str,
648 itemtype=float,
649 default={},
650 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
651 "Sources violating this limit will not be deblended. "
652 "If the fraction is `0` then the limit is a single pixel."),
653 )
655 # Size restrictions
656 maxNumberOfPeaks = pexConfig.Field(
657 dtype=int, default=200,
658 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
659 " (<= 0: unlimited)"))
660 maxFootprintArea = pexConfig.Field(
661 dtype=int, default=100_000,
662 doc=("Maximum area for footprints before they are ignored as large; "
663 "non-positive means no threshold applied"))
664 maxAreaTimesPeaks = pexConfig.Field(
665 dtype=int, default=10_000_000,
666 doc=("Maximum rectangular footprint area * nPeaks in the footprint. "
667 "This was introduced in DM-33690 to prevent fields that are crowded or have a "
668 "LSB galaxy that causes memory intensive initialization in scarlet from dominating "
669 "the overall runtime and/or causing the task to run out of memory. "
670 "(<= 0: unlimited)")
671 )
672 maxFootprintSize = pexConfig.Field(
673 dtype=int, default=0,
674 doc=("Maximum linear dimension for footprints before they are ignored "
675 "as large; non-positive means no threshold applied"))
676 minFootprintAxisRatio = pexConfig.Field(
677 dtype=float, default=0.0,
678 doc=("Minimum axis ratio for footprints before they are ignored "
679 "as large; non-positive means no threshold applied"))
680 maxSpectrumCutoff = pexConfig.Field(
681 dtype=int, default=1_000_000,
682 doc=("Maximum number of pixels * number of sources in a blend. "
683 "This is different than `maxFootprintArea` because this isn't "
684 "the footprint area but the area of the bounding box that "
685 "contains the footprint, and is also multiplied by the number of"
686 "sources in the footprint. This prevents large skinny blends with "
687 "a high density of sources from running out of memory. "
688 "If `maxSpectrumCutoff == -1` then there is no cutoff.")
689 )
690 # Failure modes
691 fallback = pexConfig.Field(
692 dtype=bool, default=True,
693 doc="Whether or not to fallback to a smaller number of components if a source does not initialize"
694 )
695 notDeblendedMask = pexConfig.Field(
696 dtype=str, default="NOT_DEBLENDED", optional=True,
697 doc="Mask name for footprints not deblended, or None")
698 catchFailures = pexConfig.Field(
699 dtype=bool, default=True,
700 doc=("If True, catch exceptions thrown by the deblender, log them, "
701 "and set a flag on the parent, instead of letting them propagate up"))
703 # Other options
704 columnInheritance = pexConfig.DictField(
705 keytype=str, itemtype=str, default={
706 "deblend_nChild": "deblend_parentNChild",
707 "deblend_nPeaks": "deblend_parentNPeaks",
708 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag",
709 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag",
710 },
711 doc="Columns to pass from the parent to the child. "
712 "The key is the name of the column for the parent record, "
713 "the value is the name of the column to use for the child."
714 )
715 pseudoColumns = pexConfig.ListField(
716 dtype=str, default=['merge_peak_sky', 'sky_source'],
717 doc="Names of flags which should never be deblended."
718 )
720 # Logging option(s)
721 loggingInterval = pexConfig.Field(
722 dtype=int, default=600,
723 doc="Interval (in seconds) to log messages (at VERBOSE level) while deblending sources.",
724 deprecated="This field is no longer used and will be removed in v25.",
725 )
726 # Testing options
727 # Some obs packages and ci packages run the full pipeline on a small
728 # subset of data to test that the pipeline is functioning properly.
729 # This is not meant as scientific validation, so it can be useful
730 # to only run on a small subset of the data that is large enough to
731 # test the desired pipeline features but not so long that the deblender
732 # is the tall pole in terms of execution times.
733 useCiLimits = pexConfig.Field(
734 dtype=bool, default=False,
735 doc="Limit the number of sources deblended for CI to prevent long build times")
736 ciDeblendChildRange = pexConfig.ListField(
737 dtype=int, default=[5, 10],
738 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
739 "If `useCiLimits==False` then this parameter is ignored.")
740 ciNumParentsToDeblend = pexConfig.Field(
741 dtype=int, default=10,
742 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
743 "within `ciDebledChildRange`. "
744 "If `useCiLimits==False` then this parameter is ignored.")
747class ScarletDeblendTask(pipeBase.Task):
748 """ScarletDeblendTask
750 Split blended sources into individual sources.
752 This task has no return value; it only modifies the SourceCatalog in-place.
753 """
754 ConfigClass = ScarletDeblendConfig
755 _DefaultName = "scarletDeblend"
757 def __init__(self, schema, peakSchema=None, **kwargs):
758 """Create the task, adding necessary fields to the given schema.
760 Parameters
761 ----------
762 schema : `lsst.afw.table.schema.schema.Schema`
763 Schema object for measurement fields; will be modified in-place.
764 peakSchema : `lsst.afw.table.schema.schema.Schema`
765 Schema of Footprint Peaks that will be passed to the deblender.
766 Any fields beyond the PeakTable minimal schema will be transferred
767 to the main source Schema. If None, no fields will be transferred
768 from the Peaks.
769 filters : list of str
770 Names of the filters used for the eposures. This is needed to store
771 the SED as a field
772 **kwargs
773 Passed to Task.__init__.
774 """
775 pipeBase.Task.__init__(self, **kwargs)
777 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
778 if peakSchema is None:
779 # In this case, the peakSchemaMapper will transfer nothing, but
780 # we'll still have one
781 # to simplify downstream code
782 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
783 else:
784 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
785 for item in peakSchema:
786 if item.key not in peakMinimalSchema:
787 self.peakSchemaMapper.addMapping(item.key, item.field)
788 # Because SchemaMapper makes a copy of the output schema
789 # you give its ctor, it isn't updating this Schema in
790 # place. That's probably a design flaw, but in the
791 # meantime, we'll keep that schema in sync with the
792 # peakSchemaMapper.getOutputSchema() manually, by adding
793 # the same fields to both.
794 schema.addField(item.field)
795 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
796 self._addSchemaKeys(schema)
797 self.schema = schema
798 self.toCopyFromParent = [item.key for item in self.schema
799 if item.field.getName().startswith("merge_footprint")]
801 def _addSchemaKeys(self, schema):
802 """Add deblender specific keys to the schema
803 """
804 # Parent (blend) fields
805 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms')
806 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge')
807 self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
808 doc='Number of children this object has (defaults to 0)')
809 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32,
810 doc="Number of initial peaks in the blend. "
811 "This includes peaks that may have been culled "
812 "during deblending or failed to deblend")
813 # Skipped flags
814 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
815 doc="Deblender skipped this source")
816 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag',
817 doc='The source has only a single peak '
818 'and was not deblended')
819 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag',
820 doc='The source is identified as a "pseudo" source and '
821 'was not deblended')
822 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
823 doc='Source had too many peaks; '
824 'only the brightest were included')
825 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
826 doc='Parent footprint covered too many pixels')
827 self.maskedKey = schema.addField('deblend_masked', type='Flag',
828 doc='Parent footprint had too many masked pixels')
829 # Convergence flags
830 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag',
831 doc='scarlet sed optimization did not converge before'
832 'config.maxIter')
833 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag',
834 doc='scarlet morph optimization did not converge before'
835 'config.maxIter')
836 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag',
837 type='Flag',
838 doc='at least one source in the blend'
839 'failed to converge')
840 # Error flags
841 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
842 doc="Deblending failed on source")
843 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25,
844 doc='Name of error if the blend failed')
845 # Deblended source fields
846 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center",
847 doc="Center used to apply constraints in scarlet",
848 unit="pixel")
849 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32,
850 doc="ID of the peak in the parent footprint. "
851 "This is not unique, but the combination of 'parent'"
852 "and 'peakId' should be for all child sources. "
853 "Top level blends with no parents have 'peakId=0'")
854 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count',
855 doc="The instFlux at the peak position of deblended mode")
856 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25,
857 doc="The type of model used, for example "
858 "MultiExtendedSource, SingleExtendedSource, PointSource")
859 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32,
860 doc="deblend_nPeaks from this records parent.")
861 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32,
862 doc="deblend_nChild from this records parent.")
863 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32,
864 doc="Flux measurement from scarlet")
865 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32,
866 doc="Final logL, used to identify regressions in scarlet.")
867 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag',
868 doc='Source had flux on the edge of the parent footprint')
869 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag',
870 doc="True when scarlet initializes sources "
871 "in the blend with a more accurate spectrum. "
872 "The algorithm uses a lot of memory, "
873 "so large dense blends will use "
874 "a less accurate initialization.")
875 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32,
876 doc="Number of components in a ScarletLiteSource. "
877 "If `config.version != 'lite'`then "
878 "this column is set to zero.")
879 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
880 doc='Deblender thought this source looked like a PSF')
881 # Blendedness/classification metrics
882 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32,
883 doc="Maximum overlap with all of the other neighbors flux "
884 "combined."
885 "This is useful as a metric for determining how blended a "
886 "source is because if it only overlaps with other sources "
887 "at or below the noise level, it is likely to be a mostly "
888 "isolated source in the deconvolved model frame.")
889 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32,
890 doc="This is the total flux from neighboring objects that "
891 "overlaps with this source.")
892 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32,
893 doc="This is the fraction of "
894 "`flux from neighbors/source flux` "
895 "for a given source within the source's"
896 "footprint.")
897 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32,
898 doc="The Bosch et al. 2018 metric for 'blendedness.' ")
900 @timeMethod
901 def run(self, mExposure, mergedSources):
902 """Get the psf from each exposure and then run deblend().
904 Parameters
905 ----------
906 mExposure : `MultibandExposure`
907 The exposures should be co-added images of the same
908 shape and region of the sky.
909 mergedSources : `SourceCatalog`
910 The merged `SourceCatalog` that contains parent footprints
911 to (potentially) deblend.
913 Returns
914 -------
915 templateCatalogs: dict
916 Keys are the names of the filters and the values are
917 `lsst.afw.table.source.source.SourceCatalog`'s.
918 These are catalogs with heavy footprints that are the templates
919 created by the multiband templates.
920 """
921 return self.deblend(mExposure, mergedSources)
923 @timeMethod
924 def deblend(self, mExposure, catalog):
925 """Deblend a data cube of multiband images
927 Parameters
928 ----------
929 mExposure : `MultibandExposure`
930 The exposures should be co-added images of the same
931 shape and region of the sky.
932 catalog : `SourceCatalog`
933 The merged `SourceCatalog` that contains parent footprints
934 to (potentially) deblend. The new deblended sources are
935 appended to this catalog in place.
937 Returns
938 -------
939 catalogs : `dict` or `None`
940 Keys are the names of the filters and the values are
941 `lsst.afw.table.source.source.SourceCatalog`'s.
942 These are catalogs with heavy footprints that are the templates
943 created by the multiband templates.
944 """
945 import time
947 # Cull footprints if required by ci
948 if self.config.useCiLimits:
949 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.",
950 len(catalog))
951 # Select parents with a number of children in the range
952 # config.ciDeblendChildRange
953 minChildren, maxChildren = self.config.ciDeblendChildRange
954 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog])
955 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
956 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
957 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range "
958 "indicated by ciDeblendChildRange. Adjust this range to include more "
959 "parents.")
960 # Keep all of the isolated parents and the first
961 # `ciNumParentsToDeblend` children
962 parents = nPeaks == 1
963 children = np.zeros((len(catalog),), dtype=bool)
964 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True
965 catalog = catalog[parents | children]
966 # We need to update the IdFactory, otherwise the the source ids
967 # will not be sequential
968 idFactory = catalog.getIdFactory()
969 maxId = np.max(catalog["id"])
970 idFactory.notify(maxId)
972 filters = mExposure.filters
973 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure))
974 periodicLog = PeriodicLogger(self.log)
976 # Create a set of wavelet coefficients if using wavelet initialization
977 if self.config.version == "lite" and self.config.morphImage == "wavelet":
978 images = mExposure.image.array
979 variance = mExposure.variance.array
980 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales)
981 else:
982 wavelets = None
984 # Add the NOT_DEBLENDED mask to the mask plane in each band
985 if self.config.notDeblendedMask:
986 for mask in mExposure.mask:
987 mask.addMaskPlane(self.config.notDeblendedMask)
989 # Initialize the persistable data model
990 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma)
991 dataModel = ScarletModelData(filters, modelPsf)
993 nParents = len(catalog)
994 nDeblendedParents = 0
995 skippedParents = []
996 for parentIndex in range(nParents):
997 parent = catalog[parentIndex]
998 foot = parent.getFootprint()
999 bbox = foot.getBBox()
1000 peaks = foot.getPeaks()
1002 # Since we use the first peak for the parent object, we should
1003 # propagate its flags to the parent source.
1004 parent.assign(peaks[0], self.peakSchemaMapper)
1006 # Block of conditions for skipping a parent with multiple children
1007 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None:
1008 self._skipParent(parent, *skipArgs)
1009 skippedParents.append(parentIndex)
1010 continue
1012 nDeblendedParents += 1
1013 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks))
1014 # Run the deblender
1015 blendError = None
1017 # Choose whether or not to use improved spectral initialization.
1018 # This significantly cuts down on the number of iterations
1019 # that the optimizer needs and usually results in a better
1020 # fit.
1021 # But using least squares on a very large blend causes memory
1022 # issues, so it is not done for large blends
1023 if self.config.setSpectra:
1024 if self.config.maxSpectrumCutoff <= 0:
1025 spectrumInit = True
1026 else:
1027 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff
1028 else:
1029 spectrumInit = False
1031 try:
1032 t0 = time.monotonic()
1033 # Build the parameter lists with the same ordering
1034 if self.config.version == "scarlet":
1035 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit)
1036 elif self.config.version == "lite":
1037 blend, skipped = deblend_lite(
1038 mExposure=mExposure,
1039 modelPsf=modelPsf,
1040 footprint=foot,
1041 config=self.config,
1042 spectrumInit=spectrumInit,
1043 wavelets=wavelets,
1044 )
1045 tf = time.monotonic()
1046 runtime = (tf-t0)*1000
1047 converged = _checkBlendConvergence(blend, self.config.relativeError)
1048 # Store the number of components in the blend
1049 if self.config.version == "lite":
1050 nComponents = len(blend.components)
1051 else:
1052 nComponents = 0
1053 nChild = len(blend.sources)
1054 # Catch all errors and filter out the ones that we know about
1055 except Exception as e:
1056 blendError = type(e).__name__
1057 if isinstance(e, ScarletGradientError):
1058 parent.set(self.iterKey, e.iterations)
1059 elif not isinstance(e, IncompleteDataError):
1060 blendError = "UnknownError"
1061 if self.config.catchFailures:
1062 # Make it easy to find UnknownErrors in the log file
1063 self.log.warn("UnknownError")
1064 import traceback
1065 traceback.print_exc()
1066 else:
1067 raise
1069 self._skipParent(
1070 parent=parent,
1071 skipKey=self.deblendFailedKey,
1072 logMessage=f"Unable to deblend source {parent.getId}: {blendError}",
1073 )
1074 parent.set(self.deblendErrorKey, blendError)
1075 skippedParents.append(parentIndex)
1076 continue
1078 # Update the parent record with the deblending results
1079 if self.config.version == "scarlet":
1080 logL = -blend.loss[-1] + blend.observations[0].log_norm
1081 elif self.config.version == "lite":
1082 logL = blend.loss[-1]
1083 self._updateParentRecord(
1084 parent=parent,
1085 nPeaks=len(peaks),
1086 nChild=nChild,
1087 nComponents=nComponents,
1088 runtime=runtime,
1089 iterations=len(blend.loss),
1090 logL=logL,
1091 spectrumInit=spectrumInit,
1092 converged=converged,
1093 )
1095 # Add each deblended source to the catalog
1096 for k, scarletSource in enumerate(blend.sources):
1097 # Skip any sources with no flux or that scarlet skipped because
1098 # it could not initialize
1099 if k in skipped or (self.config.version == "lite" and scarletSource.is_null):
1100 # No need to propagate anything
1101 continue
1102 parent.set(self.deblendSkippedKey, False)
1104 # Add all fields except the HeavyFootprint to the
1105 # source record
1106 sourceRecord = self._addChild(
1107 parent=parent,
1108 peak=scarletSource.detectedPeak,
1109 catalog=catalog,
1110 scarletSource=scarletSource,
1111 )
1112 scarletSource.recordId = sourceRecord.getId()
1113 scarletSource.peakId = scarletSource.detectedPeak.getId()
1115 # Store the blend information so that it can be persisted
1116 if self.config.version == "lite":
1117 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin())
1118 else:
1119 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin())
1120 dataModel.blends[parent.getId()] = blendData
1122 # Log a message if it has been a while since the last log.
1123 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents)
1125 # Clear the cached values in scarlet to clear out memory
1126 scarlet.cache.Cache._cache = {}
1128 # Update the mExposure mask with the footprint of skipped parents
1129 if self.config.notDeblendedMask:
1130 for mask in mExposure.mask:
1131 for parentIndex in skippedParents:
1132 fp = catalog[parentIndex].getFootprint()
1133 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
1135 self.log.info("Deblender results: of %d parent sources, %d were deblended, "
1136 "creating %d children, for a total of %d sources",
1137 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog))
1138 return catalog, dataModel
1140 def _isLargeFootprint(self, footprint):
1141 """Returns whether a Footprint is large
1143 'Large' is defined by thresholds on the area, size and axis ratio,
1144 and total area of the bounding box multiplied by
1145 the number of children.
1146 These may be disabled independently by configuring them to be
1147 non-positive.
1148 """
1149 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
1150 return True
1151 if self.config.maxFootprintSize > 0:
1152 bbox = footprint.getBBox()
1153 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
1154 return True
1155 if self.config.minFootprintAxisRatio > 0:
1156 axes = afwEll.Axes(footprint.getShape())
1157 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
1158 return True
1159 if self.config.maxAreaTimesPeaks > 0:
1160 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks:
1161 return True
1162 return False
1164 def _isMasked(self, footprint, mExposure):
1165 """Returns whether the footprint violates the mask limits
1167 Parameters
1168 ----------
1169 footprint : `lsst.afw.detection.Footprint`
1170 The footprint to check for masked pixels
1171 mMask : `lsst.afw.image.MaskX`
1172 The mask plane to check for masked pixels in the `footprint`.
1174 Returns
1175 -------
1176 isMasked : `bool`
1177 `True` if `self.config.maskPlaneLimits` is less than the
1178 fraction of pixels for a given mask in
1179 `self.config.maskLimits`.
1180 """
1181 bbox = footprint.getBBox()
1182 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0)
1183 size = float(footprint.getArea())
1184 for maskName, limit in self.config.maskLimits.items():
1185 maskVal = mExposure.mask.getPlaneBitMask(maskName)
1186 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin())
1187 # spanset of masked pixels
1188 maskedSpan = footprint.spans.intersect(_mask, maskVal)
1189 if (maskedSpan.getArea())/size > limit:
1190 return True
1191 return False
1193 def _skipParent(self, parent, skipKey, logMessage):
1194 """Update a parent record that is not being deblended.
1196 This is a fairly trivial function but is implemented to ensure
1197 that a skipped parent updates the appropriate columns
1198 consistently, and always has a flag to mark the reason that
1199 it is being skipped.
1201 Parameters
1202 ----------
1203 parent : `lsst.afw.table.source.source.SourceRecord`
1204 The parent record to flag as skipped.
1205 skipKey : `bool`
1206 The name of the flag to mark the reason for skipping.
1207 logMessage : `str`
1208 The message to display in a log.trace when a source
1209 is skipped.
1210 """
1211 if logMessage is not None:
1212 self.log.trace(logMessage)
1213 self._updateParentRecord(
1214 parent=parent,
1215 nPeaks=len(parent.getFootprint().peaks),
1216 nChild=0,
1217 nComponents=0,
1218 runtime=np.nan,
1219 iterations=0,
1220 logL=np.nan,
1221 spectrumInit=False,
1222 converged=False,
1223 )
1225 # Mark the source as skipped by the deblender and
1226 # flag the reason why.
1227 parent.set(self.deblendSkippedKey, True)
1228 parent.set(skipKey, True)
1230 def _checkSkipped(self, parent, mExposure):
1231 """Update a parent record that is not being deblended.
1233 This is a fairly trivial function but is implemented to ensure
1234 that a skipped parent updates the appropriate columns
1235 consistently, and always has a flag to mark the reason that
1236 it is being skipped.
1238 Parameters
1239 ----------
1240 parent : `lsst.afw.table.source.source.SourceRecord`
1241 The parent record to flag as skipped.
1242 mExposure : `MultibandExposure`
1243 The exposures should be co-added images of the same
1244 shape and region of the sky.
1245 Returns
1246 -------
1247 skip: `bool`
1248 `True` if the deblender will skip the parent
1249 """
1250 skipKey = None
1251 skipMessage = None
1252 footprint = parent.getFootprint()
1253 if len(footprint.peaks) < 2 and not self.config.processSingles:
1254 # Skip isolated sources unless processSingles is turned on.
1255 # Note: this does not flag isolated sources as skipped or
1256 # set the NOT_DEBLENDED mask in the exposure,
1257 # since these aren't really any skipped blends.
1258 skipKey = self.isolatedParentKey
1259 elif isPseudoSource(parent, self.config.pseudoColumns):
1260 # We also skip pseudo sources, like sky objects, which
1261 # are intended to be skipped.
1262 skipKey = self.pseudoKey
1263 if self._isLargeFootprint(footprint):
1264 # The footprint is above the maximum footprint size limit
1265 skipKey = self.tooBigKey
1266 skipMessage = f"Parent {parent.getId()}: skipping large footprint"
1267 elif self._isMasked(footprint, mExposure):
1268 # The footprint exceeds the maximum number of masked pixels
1269 skipKey = self.maskedKey
1270 skipMessage = f"Parent {parent.getId()}: skipping masked footprint"
1271 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks:
1272 # Unlike meas_deblender, in scarlet we skip the entire blend
1273 # if the number of peaks exceeds max peaks, since neglecting
1274 # to model any peaks often results in catastrophic failure
1275 # of scarlet to generate models for the brighter sources.
1276 skipKey = self.tooManyPeaksKey
1277 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks"
1278 if skipKey is not None:
1279 return (skipKey, skipMessage)
1280 return None
1282 def setSkipFlags(self, mExposure, catalog):
1283 """Set the skip flags for all of the parent sources
1285 This is mostly used for testing which parent sources will be deblended
1286 and which will be skipped based on the current configuration options.
1287 Skipped sources will have the appropriate flags set in place in the
1288 catalog.
1290 Parameters
1291 ----------
1292 mExposure : `MultibandExposure`
1293 The exposures should be co-added images of the same
1294 shape and region of the sky.
1295 catalog : `SourceCatalog`
1296 The merged `SourceCatalog` that contains parent footprints
1297 to (potentially) deblend. The new deblended sources are
1298 appended to this catalog in place.
1299 """
1300 for src in catalog:
1301 if skipArgs := self._checkSkipped(src, mExposure) is not None:
1302 self._skipParent(src, *skipArgs)
1304 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents,
1305 runtime, iterations, logL, spectrumInit, converged):
1306 """Update a parent record in all of the single band catalogs.
1308 Ensure that all locations that update a parent record,
1309 whether it is skipped or updated after deblending,
1310 update all of the appropriate columns.
1312 Parameters
1313 ----------
1314 parent : `lsst.afw.table.source.source.SourceRecord`
1315 The parent record to update.
1316 nPeaks : `int`
1317 Number of peaks in the parent footprint.
1318 nChild : `int`
1319 Number of children deblended from the parent.
1320 This may differ from `nPeaks` if some of the peaks
1321 were culled and have no deblended model.
1322 nComponents : `int`
1323 Total number of components in the parent.
1324 This is usually different than the number of children,
1325 since it is common for a single source to have multiple
1326 components.
1327 runtime : `float`
1328 Total runtime for deblending.
1329 iterations : `int`
1330 Total number of iterations in scarlet before convergence.
1331 logL : `float`
1332 Final log likelihood of the blend.
1333 spectrumInit : `bool`
1334 True when scarlet used `set_spectra` to initialize all
1335 sources with better initial intensities.
1336 converged : `bool`
1337 True when the optimizer reached convergence before
1338 reaching the maximum number of iterations.
1339 """
1340 parent.set(self.nPeaksKey, nPeaks)
1341 parent.set(self.nChildKey, nChild)
1342 parent.set(self.nComponentsKey, nComponents)
1343 parent.set(self.runtimeKey, runtime)
1344 parent.set(self.iterKey, iterations)
1345 parent.set(self.scarletLogLKey, logL)
1346 parent.set(self.scarletSpectrumInitKey, spectrumInit)
1347 parent.set(self.blendConvergenceFailedFlagKey, converged)
1349 def _addChild(self, parent, peak, catalog, scarletSource):
1350 """Add a child to a catalog.
1352 This creates a new child in the source catalog,
1353 assigning it a parent id, and adding all columns
1354 that are independent across all filter bands.
1356 Parameters
1357 ----------
1358 parent : `lsst.afw.table.source.source.SourceRecord`
1359 The parent of the new child record.
1360 peak : `lsst.afw.table.PeakRecord`
1361 The peak record for the peak from the parent peak catalog.
1362 catalog : `lsst.afw.table.source.source.SourceCatalog`
1363 The merged `SourceCatalog` that contains parent footprints
1364 to (potentially) deblend.
1365 scarletSource : `scarlet.Component`
1366 The scarlet model for the new source record.
1367 """
1368 src = catalog.addNew()
1369 for key in self.toCopyFromParent:
1370 src.set(key, parent.get(key))
1371 # The peak catalog is the same for all bands,
1372 # so we just use the first peak catalog
1373 src.assign(peak, self.peakSchemaMapper)
1374 src.setParent(parent.getId())
1375 src.set(self.nPeaksKey, 1)
1376 # Set the psf key based on whether or not the source was
1377 # deblended using the PointSource model.
1378 # This key is not that useful anymore since we now keep track of
1379 # `modelType`, but we continue to propagate it in case code downstream
1380 # is expecting it.
1381 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource")
1382 src.set(self.modelTypeKey, scarletSource.__class__.__name__)
1383 # We set the runtime to zero so that summing up the
1384 # runtime column will give the total time spent
1385 # running the deblender for the catalog.
1386 src.set(self.runtimeKey, 0)
1388 # Set the position of the peak from the parent footprint
1389 # This will make it easier to match the same source across
1390 # deblenders and across observations, where the peak
1391 # position is unlikely to change unless enough time passes
1392 # for a source to move on the sky.
1393 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"]))
1394 src.set(self.peakIdKey, peak["id"])
1396 # Store the number of components for the source
1397 src.set(self.nComponentsKey, len(scarletSource.components))
1399 # Propagate columns from the parent to the child
1400 for parentColumn, childColumn in self.config.columnInheritance.items():
1401 src.set(childColumn, parent.get(parentColumn))
1403 return src