Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%
444 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-11 02:37 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-11 02:37 -0800
1# This file is part of meas_extensions_scarlet.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from dataclasses import dataclass
23from functools import partial
24import logging
25import numpy as np
26import scarlet
27from scarlet.psf import ImagePSF, GaussianPSF
28from scarlet import Blend, Frame, Observation
29from scarlet.renderer import ConvolutionRenderer
30from scarlet.detect import get_detect_wavelets
31from scarlet.initialization import init_all_sources
32from scarlet import lite
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36from lsst.geom import Point2I, Point2D
37import lsst.afw.geom.ellipses as afwEll
38import lsst.afw.image as afwImage
39import lsst.afw.detection as afwDet
40import lsst.afw.table as afwTable
41from lsst.utils.logging import PeriodicLogger
42from lsst.utils.timer import timeMethod
43from lsst.afw.image.exposure import IncompleteDataError
45from .source import bboxToScarletBox
46from .io import ScarletModelData, scarletToData, scarletLiteToData
48# Scarlet and proxmin have a different definition of log levels than the stack,
49# so even "warnings" occur far more often than we would like.
50# So for now we only display scarlet and proxmin errors, as all other
51# scarlet outputs would be considered "TRACE" by our standards.
52scarletLogger = logging.getLogger("scarlet")
53scarletLogger.setLevel(logging.ERROR)
54proxminLogger = logging.getLogger("proxmin")
55proxminLogger.setLevel(logging.ERROR)
57__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"]
59logger = logging.getLogger(__name__)
62class ScarletGradientError(Exception):
63 """An error occurred during optimization
65 This error occurs when the optimizer encounters
66 a NaN value while calculating the gradient.
67 """
68 def __init__(self, iterations, sources):
69 self.iterations = iterations
70 self.sources = sources
71 msg = ("ScalarGradientError in iteration {0}. "
72 "NaN values introduced in sources {1}")
73 self.message = msg.format(iterations, sources)
75 def __str__(self):
76 return self.message
79def _checkBlendConvergence(blend, f_rel):
80 """Check whether or not a blend has converged
81 """
82 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1])
83 convergence = f_rel * np.abs(blend.loss[-1])
84 return deltaLoss < convergence
87def isPseudoSource(source, pseudoColumns):
88 """Check if a source is a pseudo source.
90 This is mostly for skipping sky objects,
91 but any other column can also be added to disable
92 deblending on a parent or individual source when
93 set to `True`.
95 Parameters
96 ----------
97 source : `lsst.afw.table.source.source.SourceRecord`
98 The source to check for the pseudo bit.
99 pseudoColumns : `list` of `str`
100 A list of columns to check for pseudo sources.
101 """
102 isPseudo = False
103 for col in pseudoColumns:
104 try:
105 isPseudo |= source[col]
106 except KeyError:
107 pass
108 return isPseudo
111def deblend(mExposure, footprint, config, spectrumInit):
112 """Deblend a parent footprint
114 Parameters
115 ----------
116 mExposure : `lsst.image.MultibandExposure`
117 The multiband exposure containing the image,
118 mask, and variance data.
119 footprint : `lsst.detection.Footprint`
120 The footprint of the parent to deblend.
121 config : `ScarletDeblendConfig`
122 Configuration of the deblending task.
123 spectrumInit : `bool`
124 Whether or not to initialize the spectrum.
126 Returns
127 -------
128 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData`
129 The persistable representation of a `scarlet.Blend`.
130 skipped : `list` of `int`
131 The indices of any children that failed to initialize
132 and were skipped.
133 spectrumInit : `bool`
134 Whether or not all of the sources were initialized by jointly
135 fitting their SED's. This provides a better initialization
136 but created memory issues when a blend is too large or
137 contains too many sources.
138 """
139 # Extract coordinates from each MultiColorPeak
140 bbox = footprint.getBBox()
142 # Create the data array from the masked images
143 images = mExposure.image[:, bbox].array
145 # Use the inverse variance as the weights
146 if config.useWeights:
147 weights = 1/mExposure.variance[:, bbox].array
148 else:
149 weights = np.ones_like(images)
150 badPixels = mExposure.mask.getPlaneBitMask(config.badMask)
151 mask = mExposure.mask[:, bbox].array & badPixels
152 weights[mask > 0] = 0
154 # Mask out the pixels outside the footprint
155 weights *= footprint.spans.asArray()
157 psfCenter = footprint.getCentroid()
158 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32)
159 psfs = ImagePSF(psfs)
160 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters))
162 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters)
163 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters)
164 if config.convolutionType == "fft":
165 observation.match(frame)
166 elif config.convolutionType == "real":
167 renderer = ConvolutionRenderer(observation, frame, convolution_type="real")
168 observation.match(frame, renderer=renderer)
169 else:
170 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType))
172 assert(config.sourceModel in ["single", "double", "compact", "fit"])
174 # Set the appropriate number of components
175 if config.sourceModel == "single":
176 maxComponents = 1
177 elif config.sourceModel == "double":
178 maxComponents = 2
179 elif config.sourceModel == "compact":
180 maxComponents = 0
181 elif config.sourceModel == "point":
182 raise NotImplementedError("Point source photometry is currently not implemented")
183 elif config.sourceModel == "fit":
184 # It is likely in the future that there will be some heuristic
185 # used to determine what type of model to use for each source,
186 # but that has not yet been implemented (see DM-22551)
187 raise NotImplementedError("sourceModel 'fit' has not been implemented yet")
189 # Convert the centers to pixel coordinates
190 xmin = bbox.getMinX()
191 ymin = bbox.getMinY()
192 centers = [
193 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
194 for peak in footprint.peaks
195 if not isPseudoSource(peak, config.pseudoColumns)
196 ]
198 # Only deblend sources that can be initialized
199 sources, skipped = init_all_sources(
200 frame=frame,
201 centers=centers,
202 observations=observation,
203 thresh=config.morphThresh,
204 max_components=maxComponents,
205 min_snr=config.minSNR,
206 shifting=False,
207 fallback=config.fallback,
208 silent=config.catchFailures,
209 set_spectra=spectrumInit,
210 )
212 # Attach the peak to all of the initialized sources
213 srcIndex = 0
214 for k, center in enumerate(centers):
215 if k not in skipped:
216 # This is just to make sure that there isn't a coding bug
217 assert np.all(sources[srcIndex].center == center)
218 # Store the record for the peak with the appropriate source
219 sources[srcIndex].detectedPeak = footprint.peaks[k]
220 srcIndex += 1
222 # Create the blend and attempt to optimize it
223 blend = Blend(sources, observation)
224 try:
225 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError)
226 except ArithmeticError:
227 # This occurs when a gradient update produces a NaN value
228 # This is usually due to a source initialized with a
229 # negative SED or no flux, often because the peak
230 # is a noise fluctuation in one band and not a real source.
231 iterations = len(blend.loss)
232 failedSources = []
233 for k, src in enumerate(sources):
234 if np.any(~np.isfinite(src.get_model())):
235 failedSources.append(k)
236 raise ScarletGradientError(iterations, failedSources)
238 # Store the location of the PSF center for storage
239 blend.psfCenter = (psfCenter.x, psfCenter.y)
241 return blend, skipped
244def buildLiteObservation(
245 modelPsf,
246 psfCenter,
247 mExposure,
248 footprint=None,
249 badPixelMasks=None,
250 useWeights=True,
251 convolutionType="real",
252):
253 """Generate a LiteObservation from a set of parameters.
255 Make the generation and reconstruction of a scarlet model consistent
256 by building a `LiteObservation` from a set of parameters.
258 Parameters
259 ----------
260 modelPsf : `numpy.ndarray`
261 The 2D model of the PSF in the partially deconvolved space.
262 psfCenter : `tuple` or `Point2I` or `Point2D`
263 The location `(x, y)` used as the center of the PSF.
264 mExposure : `lsst.afw.image.multiband.MultibandExposure`
265 The multi-band exposure that the model represents.
266 If `mExposure` is `None` then no image, variance, or weights are
267 attached to the observation.
268 footprint : `lsst.afw.detection.Footprint`
269 The footprint that is being fit.
270 If `Footprint` is `None` then the weights are not updated to mask
271 out pixels not contained in the footprint.
272 badPixelMasks : `list` of `str`
273 The keys from the bit mask plane used to mask out pixels
274 during the fit.
275 If `badPixelMasks` is `None` then the default values from
276 `ScarletDeblendConfig.badMask` is used.
277 useWeights : `bool`
278 Whether or not fitting should use inverse variance weights to
279 calculate the log-likelihood.
280 convolutionType : `str`
281 The type of convolution to use (either "real" or "fft").
282 When reconstructing an image it is advised to use "real" to avoid
283 polluting the footprint with
285 Returns
286 -------
287 observation : `scarlet.lite.LiteObservation`
288 The observation constructed from the input parameters.
289 """
290 # Initialize the observed PSFs
291 if not isinstance(psfCenter, Point2D):
292 psfCenter = Point2D(*psfCenter)
293 psfModels = mExposure.computePsfKernelImage(psfCenter)
295 # Use the inverse variance as the weights
296 if useWeights:
297 weights = 1/mExposure.variance.array
298 else:
299 # Mask out bad pixels
300 weights = np.ones_like(mExposure.image.array)
301 if badPixelMasks is None:
302 badPixelMasks = ScarletDeblendConfig().badMask
303 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks)
304 mask = mExposure.mask.array & badPixels
305 weights[mask > 0] = 0
307 if footprint is not None:
308 # Mask out the pixels outside the footprint
309 weights *= footprint.spans.asArray()
311 return lite.LiteObservation(
312 images=mExposure.image.array,
313 variance=mExposure.variance.array,
314 weights=weights,
315 psfs=psfModels,
316 model_psf=modelPsf[None, :, :],
317 convolution_mode=convolutionType,
318 )
321def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None):
322 """Deblend a parent footprint
324 Parameters
325 ----------
326 mExposure : `lsst.image.MultibandExposure`
327 - The multiband exposure containing the image,
328 mask, and variance data
329 footprint : `lsst.detection.Footprint`
330 - The footprint of the parent to deblend
331 config : `ScarletDeblendConfig`
332 - Configuration of the deblending task
333 """
334 # Extract coordinates from each MultiColorPeak
335 bbox = footprint.getBBox()
336 psfCenter = footprint.getCentroid()
338 observation = buildLiteObservation(
339 modelPsf=modelPsf,
340 psfCenter=psfCenter,
341 mExposure=mExposure[:, bbox],
342 footprint=footprint,
343 badPixelMasks=config.badMask,
344 useWeights=config.useWeights,
345 convolutionType=config.convolutionType,
346 )
348 # Convert the centers to pixel coordinates
349 xmin = bbox.getMinX()
350 ymin = bbox.getMinY()
351 centers = [
352 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
353 for peak in footprint.peaks
354 if not isPseudoSource(peak, config.pseudoColumns)
355 ]
357 # Initialize the sources
358 if config.morphImage == "chi2":
359 sources = lite.init_all_sources_main(
360 observation,
361 centers,
362 min_snr=config.minSNR,
363 thresh=config.morphThresh,
364 )
365 elif config.morphImage == "wavelet":
366 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin())
367 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)]
368 sources = lite.init_all_sources_wavelets(
369 observation,
370 centers,
371 use_psf=False,
372 wavelets=_wavelets,
373 min_snr=config.minSNR,
374 )
375 else:
376 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.")
378 # Set the optimizer
379 if config.optimizer == "adaprox":
380 parameterization = partial(
381 lite.init_adaprox_component,
382 bg_thresh=config.backgroundThresh,
383 max_prox_iter=config.maxProxIter,
384 )
385 elif config.optimizer == "fista":
386 parameterization = partial(
387 lite.init_fista_component,
388 bg_thresh=config.backgroundThresh,
389 )
390 else:
391 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.")
392 sources = lite.parameterize_sources(sources, observation, parameterization)
394 # Attach the peak to all of the initialized sources
395 for k, center in enumerate(centers):
396 # This is just to make sure that there isn't a coding bug
397 if len(sources[k].components) > 0 and np.any(sources[k].center != center):
398 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}")
399 # Store the record for the peak with the appropriate source
400 sources[k].detectedPeak = footprint.peaks[k]
402 blend = lite.LiteBlend(sources, observation)
404 # Initialize each source with its best fit spectrum
405 if spectrumInit:
406 blend.fit_spectra()
408 # Set the sources that could not be initialized and were skipped
409 skipped = [src for src in sources if src.is_null]
411 blend.fit(
412 max_iter=config.maxIter,
413 e_rel=config.relativeError,
414 min_iter=config.minIter,
415 reweight=False,
416 )
418 # Store the location of the PSF center for storage
419 blend.psfCenter = (psfCenter.x, psfCenter.y)
421 return blend, skipped
424@dataclass
425class DeblenderMetrics:
426 """Metrics and measurements made on single sources.
428 Store deblender metrics to be added as attributes to a scarlet source
429 before it is converted into a `SourceRecord`.
430 When DM-34414 is finished this class will be eliminated and the metrics
431 will be added to the schema using a pipeline task that calculates them
432 from the stored deconvolved models.
434 All of the parameters are one dimensional numpy arrays,
435 with an element for each band in the observed images.
437 `maxOverlap` is useful as a metric for determining how blended a source
438 is because if it only overlaps with other sources at or below
439 the noise level, it is likely to be a mostly isolated source
440 in the deconvolved model frame.
442 `fluxOverlapFraction` is potentially more useful than the canonical
443 "blendedness" (or purity) metric because it accounts for potential
444 biases created during deblending by not weighting the overlapping
445 flux with the flux of this sources model.
447 Attributes
448 ----------
449 maxOverlap : `numpy.ndarray`
450 The maximum overlap that the source has with its neighbors in
451 a single pixel.
452 fluxOverlap : `numpy.ndarray`
453 The total flux from neighbors overlapping with the current source.
454 fluxOverlapFraction : `numpy.ndarray`
455 The fraction of `flux from neighbors/source flux` for a
456 given source within the source's footprint.
457 blendedness : `numpy.ndarray`
458 The metric for determining how blended a source is using the
459 Bosch et al. 2018 metric for "blendedness." Note that some
460 surveys use the term "purity," which is `1-blendedness`.
461 """
462 maxOverlap: np.array
463 fluxOverlap: np.array
464 fluxOverlapFraction: np.array
465 blendedness: np.array
468def setDeblenderMetrics(blend):
469 """Set metrics that can be used to evalute the deblender accuracy
471 This function calculates the `DeblenderMetrics` for each source in the
472 blend, and assigns it to that sources `metrics` property in place.
474 Parameters
475 ----------
476 blend : `scarlet.lite.Blend`
477 The blend containing the sources to measure.
478 """
479 # Store the full model of the scene for comparison
480 blendModel = blend.get_model()
481 for k, src in enumerate(blend.sources):
482 # Extract the source model in the full bounding box
483 model = src.get_model(bbox=blend.bbox)
484 # The footprint is the 2D array of non-zero pixels in each band
485 footprint = np.bitwise_or.reduce(model > 0, axis=0)
486 # Calculate the metrics.
487 # See `DeblenderMetrics` for a description of each metric.
488 neighborOverlap = (blendModel-model) * footprint[None, :, :]
489 maxOverlap = np.max(neighborOverlap, axis=(1, 2))
490 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2))
491 fluxModel = np.sum(model, axis=(1, 2))
492 fluxOverlapFraction = np.zeros((len(model), ), dtype=float)
493 isFinite = fluxModel > 0
494 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite]
495 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2))
496 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness)
499class ScarletDeblendConfig(pexConfig.Config):
500 """MultibandDeblendConfig
502 Configuration for the multiband deblender.
503 The parameters are organized by the parameter types, which are
504 - Stopping Criteria: Used to determine if the fit has converged
505 - Position Fitting Criteria: Used to fit the positions of the peaks
506 - Constraints: Used to apply constraints to the peaks and their components
507 - Other: Parameters that don't fit into the above categories
508 """
509 # Stopping Criteria
510 minIter = pexConfig.Field(dtype=int, default=15,
511 doc="Minimum number of iterations before the optimizer is allowed to stop.")
512 maxIter = pexConfig.Field(dtype=int, default=300,
513 doc=("Maximum number of iterations to deblend a single parent"))
514 relativeError = pexConfig.Field(dtype=float, default=1e-2,
515 doc=("Change in the loss function between iterations to exit fitter. "
516 "Typically this is `1e-2` if measurements will be made on the "
517 "flux re-distributed models and `1e-4` when making measurements "
518 "on the models themselves."))
520 # Constraints
521 morphThresh = pexConfig.Field(dtype=float, default=1,
522 doc="Fraction of background RMS a pixel must have"
523 "to be included in the initial morphology")
524 # Lite Parameters
525 # All of these parameters (except version) are only valid if version='lite'
526 version = pexConfig.ChoiceField(
527 dtype=str,
528 default="lite",
529 allowed={
530 "scarlet": "main scarlet version (likely to be deprecated soon)",
531 "lite": "Optimized version of scarlet for survey data from a single instrument",
532 },
533 doc="The version of scarlet to use.",
534 )
535 optimizer = pexConfig.ChoiceField(
536 dtype=str,
537 default="adaprox",
538 allowed={
539 "adaprox": "Proximal ADAM optimization",
540 "fista": "Accelerated proximal gradient method",
541 },
542 doc="The optimizer to use for fitting parameters and is only used when version='lite'",
543 )
544 morphImage = pexConfig.ChoiceField(
545 dtype=str,
546 default="chi2",
547 allowed={
548 "chi2": "Initialize sources on a chi^2 image made from all available bands",
549 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image",
550 },
551 doc="The type of image to use for initializing the morphology. "
552 "Must be either 'chi2' or 'wavelet'. "
553 )
554 backgroundThresh = pexConfig.Field(
555 dtype=float,
556 default=0.25,
557 doc="Fraction of background to use for a sparsity threshold. "
558 "This prevents sources from growing unrealistically outside "
559 "the parent footprint while still modeling flux correctly "
560 "for bright sources."
561 )
562 maxProxIter = pexConfig.Field(
563 dtype=int,
564 default=1,
565 doc="Maximum number of proximal operator iterations inside of each "
566 "iteration of the optimizer. "
567 "This config field is only used if version='lite' and optimizer='adaprox'."
568 )
569 waveletScales = pexConfig.Field(
570 dtype=int,
571 default=5,
572 doc="Number of wavelet scales to use for wavelet initialization. "
573 "This field is only used when `version`='lite' and `morphImage`='wavelet'."
574 )
576 # Other scarlet paremeters
577 useWeights = pexConfig.Field(
578 dtype=bool, default=True,
579 doc=("Whether or not use use inverse variance weighting."
580 "If `useWeights` is `False` then flat weights are used"))
581 modelPsfSize = pexConfig.Field(
582 dtype=int, default=11,
583 doc="Model PSF side length in pixels")
584 modelPsfSigma = pexConfig.Field(
585 dtype=float, default=0.8,
586 doc="Define sigma for the model frame PSF")
587 minSNR = pexConfig.Field(
588 dtype=float, default=50,
589 doc="Minimum Signal to noise to accept the source."
590 "Sources with lower flux will be initialized with the PSF but updated "
591 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).")
592 saveTemplates = pexConfig.Field(
593 dtype=bool, default=True,
594 doc="Whether or not to save the SEDs and templates")
595 processSingles = pexConfig.Field(
596 dtype=bool, default=True,
597 doc="Whether or not to process isolated sources in the deblender")
598 convolutionType = pexConfig.Field(
599 dtype=str, default="fft",
600 doc="Type of convolution to render the model to the observations.\n"
601 "- 'fft': perform convolutions in Fourier space\n"
602 "- 'real': peform convolutions in real space.")
603 sourceModel = pexConfig.Field(
604 dtype=str, default="double",
605 doc=("How to determine which model to use for sources, from\n"
606 "- 'single': use a single component for all sources\n"
607 "- 'double': use a bulge disk model for all sources\n"
608 "- 'compact': use a single component model, initialzed with a point source morphology, "
609 " for all sources\n"
610 "- 'point': use a point-source model for all sources\n"
611 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"),
612 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.",
613 )
614 setSpectra = pexConfig.Field(
615 dtype=bool, default=True,
616 doc="Whether or not to solve for the best-fit spectra during initialization. "
617 "This makes initialization slightly longer, as it requires a convolution "
618 "to set the optimal spectra, but results in a much better initial log-likelihood "
619 "and reduced total runtime, with convergence in fewer iterations."
620 "This option is only used when "
621 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.")
623 # Mask-plane restrictions
624 badMask = pexConfig.ListField(
625 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"],
626 doc="Whether or not to process isolated sources in the deblender")
627 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
628 doc="Mask planes to ignore when performing statistics")
629 maskLimits = pexConfig.DictField(
630 keytype=str,
631 itemtype=float,
632 default={},
633 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
634 "Sources violating this limit will not be deblended. "
635 "If the fraction is `0` then the limit is a single pixel."),
636 )
638 # Size restrictions
639 maxNumberOfPeaks = pexConfig.Field(
640 dtype=int, default=200,
641 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
642 " (<= 0: unlimited)"))
643 maxFootprintArea = pexConfig.Field(
644 dtype=int, default=100_000,
645 doc=("Maximum area for footprints before they are ignored as large; "
646 "non-positive means no threshold applied"))
647 maxAreaTimesPeaks = pexConfig.Field(
648 dtype=int, default=10_000_000,
649 doc=("Maximum rectangular footprint area * nPeaks in the footprint. "
650 "This was introduced in DM-33690 to prevent fields that are crowded or have a "
651 "LSB galaxy that causes memory intensive initialization in scarlet from dominating "
652 "the overall runtime and/or causing the task to run out of memory. "
653 "(<= 0: unlimited)")
654 )
655 maxFootprintSize = pexConfig.Field(
656 dtype=int, default=0,
657 doc=("Maximum linear dimension for footprints before they are ignored "
658 "as large; non-positive means no threshold applied"))
659 minFootprintAxisRatio = pexConfig.Field(
660 dtype=float, default=0.0,
661 doc=("Minimum axis ratio for footprints before they are ignored "
662 "as large; non-positive means no threshold applied"))
663 maxSpectrumCutoff = pexConfig.Field(
664 dtype=int, default=1_000_000,
665 doc=("Maximum number of pixels * number of sources in a blend. "
666 "This is different than `maxFootprintArea` because this isn't "
667 "the footprint area but the area of the bounding box that "
668 "contains the footprint, and is also multiplied by the number of"
669 "sources in the footprint. This prevents large skinny blends with "
670 "a high density of sources from running out of memory. "
671 "If `maxSpectrumCutoff == -1` then there is no cutoff.")
672 )
673 # Failure modes
674 fallback = pexConfig.Field(
675 dtype=bool, default=True,
676 doc="Whether or not to fallback to a smaller number of components if a source does not initialize"
677 )
678 notDeblendedMask = pexConfig.Field(
679 dtype=str, default="NOT_DEBLENDED", optional=True,
680 doc="Mask name for footprints not deblended, or None")
681 catchFailures = pexConfig.Field(
682 dtype=bool, default=True,
683 doc=("If True, catch exceptions thrown by the deblender, log them, "
684 "and set a flag on the parent, instead of letting them propagate up"))
686 # Other options
687 columnInheritance = pexConfig.DictField(
688 keytype=str, itemtype=str, default={
689 "deblend_nChild": "deblend_parentNChild",
690 "deblend_nPeaks": "deblend_parentNPeaks",
691 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag",
692 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag",
693 },
694 doc="Columns to pass from the parent to the child. "
695 "The key is the name of the column for the parent record, "
696 "the value is the name of the column to use for the child."
697 )
698 pseudoColumns = pexConfig.ListField(
699 dtype=str, default=['merge_peak_sky', 'sky_source'],
700 doc="Names of flags which should never be deblended."
701 )
703 # Testing options
704 # Some obs packages and ci packages run the full pipeline on a small
705 # subset of data to test that the pipeline is functioning properly.
706 # This is not meant as scientific validation, so it can be useful
707 # to only run on a small subset of the data that is large enough to
708 # test the desired pipeline features but not so long that the deblender
709 # is the tall pole in terms of execution times.
710 useCiLimits = pexConfig.Field(
711 dtype=bool, default=False,
712 doc="Limit the number of sources deblended for CI to prevent long build times")
713 ciDeblendChildRange = pexConfig.ListField(
714 dtype=int, default=[5, 10],
715 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
716 "If `useCiLimits==False` then this parameter is ignored.")
717 ciNumParentsToDeblend = pexConfig.Field(
718 dtype=int, default=10,
719 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
720 "within `ciDebledChildRange`. "
721 "If `useCiLimits==False` then this parameter is ignored.")
724class ScarletDeblendTask(pipeBase.Task):
725 """ScarletDeblendTask
727 Split blended sources into individual sources.
729 This task has no return value; it only modifies the SourceCatalog in-place.
730 """
731 ConfigClass = ScarletDeblendConfig
732 _DefaultName = "scarletDeblend"
734 def __init__(self, schema, peakSchema=None, **kwargs):
735 """Create the task, adding necessary fields to the given schema.
737 Parameters
738 ----------
739 schema : `lsst.afw.table.schema.schema.Schema`
740 Schema object for measurement fields; will be modified in-place.
741 peakSchema : `lsst.afw.table.schema.schema.Schema`
742 Schema of Footprint Peaks that will be passed to the deblender.
743 Any fields beyond the PeakTable minimal schema will be transferred
744 to the main source Schema. If None, no fields will be transferred
745 from the Peaks.
746 filters : list of str
747 Names of the filters used for the eposures. This is needed to store
748 the SED as a field
749 **kwargs
750 Passed to Task.__init__.
751 """
752 pipeBase.Task.__init__(self, **kwargs)
754 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
755 if peakSchema is None:
756 # In this case, the peakSchemaMapper will transfer nothing, but
757 # we'll still have one
758 # to simplify downstream code
759 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
760 else:
761 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
762 for item in peakSchema:
763 if item.key not in peakMinimalSchema:
764 self.peakSchemaMapper.addMapping(item.key, item.field)
765 # Because SchemaMapper makes a copy of the output schema
766 # you give its ctor, it isn't updating this Schema in
767 # place. That's probably a design flaw, but in the
768 # meantime, we'll keep that schema in sync with the
769 # peakSchemaMapper.getOutputSchema() manually, by adding
770 # the same fields to both.
771 schema.addField(item.field)
772 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
773 self._addSchemaKeys(schema)
774 self.schema = schema
775 self.toCopyFromParent = [item.key for item in self.schema
776 if item.field.getName().startswith("merge_footprint")]
778 def _addSchemaKeys(self, schema):
779 """Add deblender specific keys to the schema
780 """
781 # Parent (blend) fields
782 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms')
783 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge')
784 self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
785 doc='Number of children this object has (defaults to 0)')
786 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32,
787 doc="Number of initial peaks in the blend. "
788 "This includes peaks that may have been culled "
789 "during deblending or failed to deblend")
790 # Skipped flags
791 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
792 doc="Deblender skipped this source")
793 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag',
794 doc='The source has only a single peak '
795 'and was not deblended')
796 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag',
797 doc='The source is identified as a "pseudo" source and '
798 'was not deblended')
799 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
800 doc='Source had too many peaks; '
801 'only the brightest were included')
802 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
803 doc='Parent footprint covered too many pixels')
804 self.maskedKey = schema.addField('deblend_masked', type='Flag',
805 doc='Parent footprint had too many masked pixels')
806 # Convergence flags
807 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag',
808 doc='scarlet sed optimization did not converge before'
809 'config.maxIter')
810 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag',
811 doc='scarlet morph optimization did not converge before'
812 'config.maxIter')
813 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag',
814 type='Flag',
815 doc='at least one source in the blend'
816 'failed to converge')
817 # Error flags
818 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
819 doc="Deblending failed on source")
820 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25,
821 doc='Name of error if the blend failed')
822 # Deblended source fields
823 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center",
824 doc="Center used to apply constraints in scarlet",
825 unit="pixel")
826 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32,
827 doc="ID of the peak in the parent footprint. "
828 "This is not unique, but the combination of 'parent'"
829 "and 'peakId' should be for all child sources. "
830 "Top level blends with no parents have 'peakId=0'")
831 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count',
832 doc="The instFlux at the peak position of deblended mode")
833 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25,
834 doc="The type of model used, for example "
835 "MultiExtendedSource, SingleExtendedSource, PointSource")
836 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32,
837 doc="deblend_nPeaks from this records parent.")
838 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32,
839 doc="deblend_nChild from this records parent.")
840 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32,
841 doc="Flux measurement from scarlet")
842 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32,
843 doc="Final logL, used to identify regressions in scarlet.")
844 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag',
845 doc='Source had flux on the edge of the parent footprint')
846 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag',
847 doc="True when scarlet initializes sources "
848 "in the blend with a more accurate spectrum. "
849 "The algorithm uses a lot of memory, "
850 "so large dense blends will use "
851 "a less accurate initialization.")
852 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32,
853 doc="Number of components in a ScarletLiteSource. "
854 "If `config.version != 'lite'`then "
855 "this column is set to zero.")
856 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
857 doc='Deblender thought this source looked like a PSF')
858 # Blendedness/classification metrics
859 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32,
860 doc="Maximum overlap with all of the other neighbors flux "
861 "combined."
862 "This is useful as a metric for determining how blended a "
863 "source is because if it only overlaps with other sources "
864 "at or below the noise level, it is likely to be a mostly "
865 "isolated source in the deconvolved model frame.")
866 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32,
867 doc="This is the total flux from neighboring objects that "
868 "overlaps with this source.")
869 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32,
870 doc="This is the fraction of "
871 "`flux from neighbors/source flux` "
872 "for a given source within the source's"
873 "footprint.")
874 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32,
875 doc="The Bosch et al. 2018 metric for 'blendedness.' ")
877 @timeMethod
878 def run(self, mExposure, mergedSources):
879 """Get the psf from each exposure and then run deblend().
881 Parameters
882 ----------
883 mExposure : `MultibandExposure`
884 The exposures should be co-added images of the same
885 shape and region of the sky.
886 mergedSources : `SourceCatalog`
887 The merged `SourceCatalog` that contains parent footprints
888 to (potentially) deblend.
890 Returns
891 -------
892 templateCatalogs: dict
893 Keys are the names of the filters and the values are
894 `lsst.afw.table.source.source.SourceCatalog`'s.
895 These are catalogs with heavy footprints that are the templates
896 created by the multiband templates.
897 """
898 return self.deblend(mExposure, mergedSources)
900 @timeMethod
901 def deblend(self, mExposure, catalog):
902 """Deblend a data cube of multiband images
904 Parameters
905 ----------
906 mExposure : `MultibandExposure`
907 The exposures should be co-added images of the same
908 shape and region of the sky.
909 catalog : `SourceCatalog`
910 The merged `SourceCatalog` that contains parent footprints
911 to (potentially) deblend. The new deblended sources are
912 appended to this catalog in place.
914 Returns
915 -------
916 catalogs : `dict` or `None`
917 Keys are the names of the filters and the values are
918 `lsst.afw.table.source.source.SourceCatalog`'s.
919 These are catalogs with heavy footprints that are the templates
920 created by the multiband templates.
921 """
922 import time
924 # Cull footprints if required by ci
925 if self.config.useCiLimits:
926 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.",
927 len(catalog))
928 # Select parents with a number of children in the range
929 # config.ciDeblendChildRange
930 minChildren, maxChildren = self.config.ciDeblendChildRange
931 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog])
932 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
933 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
934 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range "
935 "indicated by ciDeblendChildRange. Adjust this range to include more "
936 "parents.")
937 # Keep all of the isolated parents and the first
938 # `ciNumParentsToDeblend` children
939 parents = nPeaks == 1
940 children = np.zeros((len(catalog),), dtype=bool)
941 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True
942 catalog = catalog[parents | children]
943 # We need to update the IdFactory, otherwise the the source ids
944 # will not be sequential
945 idFactory = catalog.getIdFactory()
946 maxId = np.max(catalog["id"])
947 idFactory.notify(maxId)
949 filters = mExposure.filters
950 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure))
951 periodicLog = PeriodicLogger(self.log)
953 # Create a set of wavelet coefficients if using wavelet initialization
954 if self.config.version == "lite" and self.config.morphImage == "wavelet":
955 images = mExposure.image.array
956 variance = mExposure.variance.array
957 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales)
958 else:
959 wavelets = None
961 # Add the NOT_DEBLENDED mask to the mask plane in each band
962 if self.config.notDeblendedMask:
963 for mask in mExposure.mask:
964 mask.addMaskPlane(self.config.notDeblendedMask)
966 # Initialize the persistable data model
967 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma)
968 dataModel = ScarletModelData(filters, modelPsf)
970 nParents = len(catalog)
971 nDeblendedParents = 0
972 skippedParents = []
973 for parentIndex in range(nParents):
974 parent = catalog[parentIndex]
975 foot = parent.getFootprint()
976 bbox = foot.getBBox()
977 peaks = foot.getPeaks()
979 # Since we use the first peak for the parent object, we should
980 # propagate its flags to the parent source.
981 parent.assign(peaks[0], self.peakSchemaMapper)
983 # Block of conditions for skipping a parent with multiple children
984 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None:
985 self._skipParent(parent, *skipArgs)
986 skippedParents.append(parentIndex)
987 continue
989 nDeblendedParents += 1
990 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks))
991 # Run the deblender
992 blendError = None
994 # Choose whether or not to use improved spectral initialization.
995 # This significantly cuts down on the number of iterations
996 # that the optimizer needs and usually results in a better
997 # fit.
998 # But using least squares on a very large blend causes memory
999 # issues, so it is not done for large blends
1000 if self.config.setSpectra:
1001 if self.config.maxSpectrumCutoff <= 0:
1002 spectrumInit = True
1003 else:
1004 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff
1005 else:
1006 spectrumInit = False
1008 try:
1009 t0 = time.monotonic()
1010 # Build the parameter lists with the same ordering
1011 if self.config.version == "scarlet":
1012 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit)
1013 elif self.config.version == "lite":
1014 blend, skipped = deblend_lite(
1015 mExposure=mExposure,
1016 modelPsf=modelPsf,
1017 footprint=foot,
1018 config=self.config,
1019 spectrumInit=spectrumInit,
1020 wavelets=wavelets,
1021 )
1022 tf = time.monotonic()
1023 runtime = (tf-t0)*1000
1024 converged = _checkBlendConvergence(blend, self.config.relativeError)
1025 # Store the number of components in the blend
1026 if self.config.version == "lite":
1027 nComponents = len(blend.components)
1028 else:
1029 nComponents = 0
1030 nChild = len(blend.sources)
1031 # Catch all errors and filter out the ones that we know about
1032 except Exception as e:
1033 blendError = type(e).__name__
1034 if isinstance(e, ScarletGradientError):
1035 parent.set(self.iterKey, e.iterations)
1036 elif not isinstance(e, IncompleteDataError):
1037 blendError = "UnknownError"
1038 if self.config.catchFailures:
1039 # Make it easy to find UnknownErrors in the log file
1040 self.log.warn("UnknownError")
1041 import traceback
1042 traceback.print_exc()
1043 else:
1044 raise
1046 self._skipParent(
1047 parent=parent,
1048 skipKey=self.deblendFailedKey,
1049 logMessage=f"Unable to deblend source {parent.getId}: {blendError}",
1050 )
1051 parent.set(self.deblendErrorKey, blendError)
1052 skippedParents.append(parentIndex)
1053 continue
1055 # Update the parent record with the deblending results
1056 if self.config.version == "scarlet":
1057 logL = -blend.loss[-1] + blend.observations[0].log_norm
1058 elif self.config.version == "lite":
1059 logL = blend.loss[-1]
1060 self._updateParentRecord(
1061 parent=parent,
1062 nPeaks=len(peaks),
1063 nChild=nChild,
1064 nComponents=nComponents,
1065 runtime=runtime,
1066 iterations=len(blend.loss),
1067 logL=logL,
1068 spectrumInit=spectrumInit,
1069 converged=converged,
1070 )
1072 # Add each deblended source to the catalog
1073 for k, scarletSource in enumerate(blend.sources):
1074 # Skip any sources with no flux or that scarlet skipped because
1075 # it could not initialize
1076 if k in skipped or (self.config.version == "lite" and scarletSource.is_null):
1077 # No need to propagate anything
1078 continue
1079 parent.set(self.deblendSkippedKey, False)
1081 # Add all fields except the HeavyFootprint to the
1082 # source record
1083 sourceRecord = self._addChild(
1084 parent=parent,
1085 peak=scarletSource.detectedPeak,
1086 catalog=catalog,
1087 scarletSource=scarletSource,
1088 )
1089 scarletSource.recordId = sourceRecord.getId()
1090 scarletSource.peakId = scarletSource.detectedPeak.getId()
1092 # Store the blend information so that it can be persisted
1093 if self.config.version == "lite":
1094 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin())
1095 else:
1096 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin())
1097 dataModel.blends[parent.getId()] = blendData
1099 # Log a message if it has been a while since the last log.
1100 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents)
1102 # Clear the cached values in scarlet to clear out memory
1103 scarlet.cache.Cache._cache = {}
1105 # Update the mExposure mask with the footprint of skipped parents
1106 if self.config.notDeblendedMask:
1107 for mask in mExposure.mask:
1108 for parentIndex in skippedParents:
1109 fp = catalog[parentIndex].getFootprint()
1110 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
1112 self.log.info("Deblender results: of %d parent sources, %d were deblended, "
1113 "creating %d children, for a total of %d sources",
1114 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog))
1115 return catalog, dataModel
1117 def _isLargeFootprint(self, footprint):
1118 """Returns whether a Footprint is large
1120 'Large' is defined by thresholds on the area, size and axis ratio,
1121 and total area of the bounding box multiplied by
1122 the number of children.
1123 These may be disabled independently by configuring them to be
1124 non-positive.
1125 """
1126 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
1127 return True
1128 if self.config.maxFootprintSize > 0:
1129 bbox = footprint.getBBox()
1130 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
1131 return True
1132 if self.config.minFootprintAxisRatio > 0:
1133 axes = afwEll.Axes(footprint.getShape())
1134 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
1135 return True
1136 if self.config.maxAreaTimesPeaks > 0:
1137 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks:
1138 return True
1139 return False
1141 def _isMasked(self, footprint, mExposure):
1142 """Returns whether the footprint violates the mask limits
1144 Parameters
1145 ----------
1146 footprint : `lsst.afw.detection.Footprint`
1147 The footprint to check for masked pixels
1148 mMask : `lsst.afw.image.MaskX`
1149 The mask plane to check for masked pixels in the `footprint`.
1151 Returns
1152 -------
1153 isMasked : `bool`
1154 `True` if `self.config.maskPlaneLimits` is less than the
1155 fraction of pixels for a given mask in
1156 `self.config.maskLimits`.
1157 """
1158 bbox = footprint.getBBox()
1159 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0)
1160 size = float(footprint.getArea())
1161 for maskName, limit in self.config.maskLimits.items():
1162 maskVal = mExposure.mask.getPlaneBitMask(maskName)
1163 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin())
1164 # spanset of masked pixels
1165 maskedSpan = footprint.spans.intersect(_mask, maskVal)
1166 if (maskedSpan.getArea())/size > limit:
1167 return True
1168 return False
1170 def _skipParent(self, parent, skipKey, logMessage):
1171 """Update a parent record that is not being deblended.
1173 This is a fairly trivial function but is implemented to ensure
1174 that a skipped parent updates the appropriate columns
1175 consistently, and always has a flag to mark the reason that
1176 it is being skipped.
1178 Parameters
1179 ----------
1180 parent : `lsst.afw.table.source.source.SourceRecord`
1181 The parent record to flag as skipped.
1182 skipKey : `bool`
1183 The name of the flag to mark the reason for skipping.
1184 logMessage : `str`
1185 The message to display in a log.trace when a source
1186 is skipped.
1187 """
1188 if logMessage is not None:
1189 self.log.trace(logMessage)
1190 self._updateParentRecord(
1191 parent=parent,
1192 nPeaks=len(parent.getFootprint().peaks),
1193 nChild=0,
1194 nComponents=0,
1195 runtime=np.nan,
1196 iterations=0,
1197 logL=np.nan,
1198 spectrumInit=False,
1199 converged=False,
1200 )
1202 # Mark the source as skipped by the deblender and
1203 # flag the reason why.
1204 parent.set(self.deblendSkippedKey, True)
1205 parent.set(skipKey, True)
1207 def _checkSkipped(self, parent, mExposure):
1208 """Update a parent record that is not being deblended.
1210 This is a fairly trivial function but is implemented to ensure
1211 that a skipped parent updates the appropriate columns
1212 consistently, and always has a flag to mark the reason that
1213 it is being skipped.
1215 Parameters
1216 ----------
1217 parent : `lsst.afw.table.source.source.SourceRecord`
1218 The parent record to flag as skipped.
1219 mExposure : `MultibandExposure`
1220 The exposures should be co-added images of the same
1221 shape and region of the sky.
1222 Returns
1223 -------
1224 skip: `bool`
1225 `True` if the deblender will skip the parent
1226 """
1227 skipKey = None
1228 skipMessage = None
1229 footprint = parent.getFootprint()
1230 if len(footprint.peaks) < 2 and not self.config.processSingles:
1231 # Skip isolated sources unless processSingles is turned on.
1232 # Note: this does not flag isolated sources as skipped or
1233 # set the NOT_DEBLENDED mask in the exposure,
1234 # since these aren't really any skipped blends.
1235 skipKey = self.isolatedParentKey
1236 elif isPseudoSource(parent, self.config.pseudoColumns):
1237 # We also skip pseudo sources, like sky objects, which
1238 # are intended to be skipped.
1239 skipKey = self.pseudoKey
1240 if self._isLargeFootprint(footprint):
1241 # The footprint is above the maximum footprint size limit
1242 skipKey = self.tooBigKey
1243 skipMessage = f"Parent {parent.getId()}: skipping large footprint"
1244 elif self._isMasked(footprint, mExposure):
1245 # The footprint exceeds the maximum number of masked pixels
1246 skipKey = self.maskedKey
1247 skipMessage = f"Parent {parent.getId()}: skipping masked footprint"
1248 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks:
1249 # Unlike meas_deblender, in scarlet we skip the entire blend
1250 # if the number of peaks exceeds max peaks, since neglecting
1251 # to model any peaks often results in catastrophic failure
1252 # of scarlet to generate models for the brighter sources.
1253 skipKey = self.tooManyPeaksKey
1254 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks"
1255 if skipKey is not None:
1256 return (skipKey, skipMessage)
1257 return None
1259 def setSkipFlags(self, mExposure, catalog):
1260 """Set the skip flags for all of the parent sources
1262 This is mostly used for testing which parent sources will be deblended
1263 and which will be skipped based on the current configuration options.
1264 Skipped sources will have the appropriate flags set in place in the
1265 catalog.
1267 Parameters
1268 ----------
1269 mExposure : `MultibandExposure`
1270 The exposures should be co-added images of the same
1271 shape and region of the sky.
1272 catalog : `SourceCatalog`
1273 The merged `SourceCatalog` that contains parent footprints
1274 to (potentially) deblend. The new deblended sources are
1275 appended to this catalog in place.
1276 """
1277 for src in catalog:
1278 if skipArgs := self._checkSkipped(src, mExposure) is not None:
1279 self._skipParent(src, *skipArgs)
1281 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents,
1282 runtime, iterations, logL, spectrumInit, converged):
1283 """Update a parent record in all of the single band catalogs.
1285 Ensure that all locations that update a parent record,
1286 whether it is skipped or updated after deblending,
1287 update all of the appropriate columns.
1289 Parameters
1290 ----------
1291 parent : `lsst.afw.table.source.source.SourceRecord`
1292 The parent record to update.
1293 nPeaks : `int`
1294 Number of peaks in the parent footprint.
1295 nChild : `int`
1296 Number of children deblended from the parent.
1297 This may differ from `nPeaks` if some of the peaks
1298 were culled and have no deblended model.
1299 nComponents : `int`
1300 Total number of components in the parent.
1301 This is usually different than the number of children,
1302 since it is common for a single source to have multiple
1303 components.
1304 runtime : `float`
1305 Total runtime for deblending.
1306 iterations : `int`
1307 Total number of iterations in scarlet before convergence.
1308 logL : `float`
1309 Final log likelihood of the blend.
1310 spectrumInit : `bool`
1311 True when scarlet used `set_spectra` to initialize all
1312 sources with better initial intensities.
1313 converged : `bool`
1314 True when the optimizer reached convergence before
1315 reaching the maximum number of iterations.
1316 """
1317 parent.set(self.nPeaksKey, nPeaks)
1318 parent.set(self.nChildKey, nChild)
1319 parent.set(self.nComponentsKey, nComponents)
1320 parent.set(self.runtimeKey, runtime)
1321 parent.set(self.iterKey, iterations)
1322 parent.set(self.scarletLogLKey, logL)
1323 parent.set(self.scarletSpectrumInitKey, spectrumInit)
1324 parent.set(self.blendConvergenceFailedFlagKey, converged)
1326 def _addChild(self, parent, peak, catalog, scarletSource):
1327 """Add a child to a catalog.
1329 This creates a new child in the source catalog,
1330 assigning it a parent id, and adding all columns
1331 that are independent across all filter bands.
1333 Parameters
1334 ----------
1335 parent : `lsst.afw.table.source.source.SourceRecord`
1336 The parent of the new child record.
1337 peak : `lsst.afw.table.PeakRecord`
1338 The peak record for the peak from the parent peak catalog.
1339 catalog : `lsst.afw.table.source.source.SourceCatalog`
1340 The merged `SourceCatalog` that contains parent footprints
1341 to (potentially) deblend.
1342 scarletSource : `scarlet.Component`
1343 The scarlet model for the new source record.
1344 """
1345 src = catalog.addNew()
1346 for key in self.toCopyFromParent:
1347 src.set(key, parent.get(key))
1348 # The peak catalog is the same for all bands,
1349 # so we just use the first peak catalog
1350 src.assign(peak, self.peakSchemaMapper)
1351 src.setParent(parent.getId())
1352 src.set(self.nPeaksKey, 1)
1353 # Set the psf key based on whether or not the source was
1354 # deblended using the PointSource model.
1355 # This key is not that useful anymore since we now keep track of
1356 # `modelType`, but we continue to propagate it in case code downstream
1357 # is expecting it.
1358 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource")
1359 src.set(self.modelTypeKey, scarletSource.__class__.__name__)
1360 # We set the runtime to zero so that summing up the
1361 # runtime column will give the total time spent
1362 # running the deblender for the catalog.
1363 src.set(self.runtimeKey, 0)
1365 # Set the position of the peak from the parent footprint
1366 # This will make it easier to match the same source across
1367 # deblenders and across observations, where the peak
1368 # position is unlikely to change unless enough time passes
1369 # for a source to move on the sky.
1370 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"]))
1371 src.set(self.peakIdKey, peak["id"])
1373 # Store the number of components for the source
1374 src.set(self.nComponentsKey, len(scarletSource.components))
1376 # Propagate columns from the parent to the child
1377 for parentColumn, childColumn in self.config.columnInheritance.items():
1378 src.set(childColumn, parent.get(parentColumn))
1380 return src