Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%
460 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 16:48 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 16:48 +0000
1# This file is part of meas_extensions_scarlet.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from dataclasses import dataclass
23from functools import partial
24import logging
25import numpy as np
26import scarlet
27from scarlet.psf import ImagePSF, GaussianPSF
28from scarlet import Blend, Frame, Observation
29from scarlet.renderer import ConvolutionRenderer
30from scarlet.detect import get_detect_wavelets
31from scarlet.initialization import init_all_sources
32from scarlet import lite
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.geom as geom
37import lsst.afw.geom.ellipses as afwEll
38import lsst.afw.image as afwImage
39import lsst.afw.detection as afwDet
40import lsst.afw.table as afwTable
41from lsst.utils.logging import PeriodicLogger
42from lsst.utils.timer import timeMethod
43from lsst.afw.image import IncompleteDataError
45from .source import bboxToScarletBox
46from .io import ScarletModelData, scarletToData, scarletLiteToData
48# Scarlet and proxmin have a different definition of log levels than the stack,
49# so even "warnings" occur far more often than we would like.
50# So for now we only display scarlet and proxmin errors, as all other
51# scarlet outputs would be considered "TRACE" by our standards.
52scarletLogger = logging.getLogger("scarlet")
53scarletLogger.setLevel(logging.ERROR)
54proxminLogger = logging.getLogger("proxmin")
55proxminLogger.setLevel(logging.ERROR)
57__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"]
59logger = logging.getLogger(__name__)
62class ScarletGradientError(Exception):
63 """An error occurred during optimization
65 This error occurs when the optimizer encounters
66 a NaN value while calculating the gradient.
67 """
68 def __init__(self, iterations, sources):
69 self.iterations = iterations
70 self.sources = sources
71 msg = ("ScalarGradientError in iteration {0}. "
72 "NaN values introduced in sources {1}")
73 self.message = msg.format(iterations, sources)
75 def __str__(self):
76 return self.message
79def _checkBlendConvergence(blend, f_rel):
80 """Check whether or not a blend has converged
81 """
82 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1])
83 convergence = f_rel * np.abs(blend.loss[-1])
84 return deltaLoss < convergence
87def isPseudoSource(source, pseudoColumns):
88 """Check if a source is a pseudo source.
90 This is mostly for skipping sky objects,
91 but any other column can also be added to disable
92 deblending on a parent or individual source when
93 set to `True`.
95 Parameters
96 ----------
97 source : `lsst.afw.table.source.source.SourceRecord`
98 The source to check for the pseudo bit.
99 pseudoColumns : `list` of `str`
100 A list of columns to check for pseudo sources.
101 """
102 isPseudo = False
103 for col in pseudoColumns:
104 try:
105 isPseudo |= source[col]
106 except KeyError:
107 pass
108 return isPseudo
111def computePsfKernelImage(mExposure, psfCenter):
112 """Compute the PSF kernel image and update the multiband exposure
113 if not all of the PSF images could be computed.
115 Parameters
116 ----------
117 psfCenter : `tuple` or `Point2I` or `Point2D`
118 The location `(x, y)` used as the center of the PSF.
120 Returns
121 -------
122 psfModels : `np.ndarray`
123 The multiband PSF image
124 mExposure : `MultibandExposure`
125 The exposure, updated to only use bands that
126 successfully generated a PSF image.
127 """
128 if not isinstance(psfCenter, geom.Point2D):
129 psfCenter = geom.Point2D(*psfCenter)
131 try:
132 psfModels = mExposure.computePsfKernelImage(psfCenter)
133 except IncompleteDataError as e:
134 psfModels = e.partialPsf
135 # Use only the bands that successfully generated a PSF image.
136 bands = psfModels.filters
137 mExposure = mExposure[bands,]
138 if len(bands) == 1:
139 # Only a single band generated a PSF, so the MultibandExposure
140 # became a single band ExposureF.
141 # Convert the result back into a MultibandExposure.
142 mExposure = afwImage.MultibandExposure.fromExposures(bands, [mExposure])
143 return psfModels.array, mExposure
146def deblend(mExposure, footprint, config, spectrumInit):
147 """Deblend a parent footprint
149 Parameters
150 ----------
151 mExposure : `lsst.image.MultibandExposure`
152 The multiband exposure containing the image,
153 mask, and variance data.
154 footprint : `lsst.detection.Footprint`
155 The footprint of the parent to deblend.
156 config : `ScarletDeblendConfig`
157 Configuration of the deblending task.
158 spectrumInit : `bool`
159 Whether or not to initialize the spectrum.
161 Returns
162 -------
163 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData`
164 The persistable representation of a `scarlet.Blend`.
165 skipped : `list` of `int`
166 The indices of any children that failed to initialize
167 and were skipped.
168 """
169 # Extract coordinates from each MultiColorPeak
170 bbox = footprint.getBBox()
172 # Create the data array from the masked images
173 images = mExposure.image[:, bbox].array
175 # Use the inverse variance as the weights
176 if config.useWeights:
177 weights = 1/mExposure.variance[:, bbox].array
178 else:
179 weights = np.ones_like(images)
180 badPixels = mExposure.mask.getPlaneBitMask(config.badMask)
181 mask = mExposure.mask[:, bbox].array & badPixels
182 weights[mask > 0] = 0
184 # Mask out the pixels outside the footprint
185 weights *= footprint.spans.asArray()
187 psfCenter = footprint.getCentroid()
188 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32)
189 psfs = ImagePSF(psfs)
190 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters))
192 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters)
193 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters)
194 if config.convolutionType == "fft":
195 observation.match(frame)
196 elif config.convolutionType == "real":
197 renderer = ConvolutionRenderer(observation, frame, convolution_type="real")
198 observation.match(frame, renderer=renderer)
199 else:
200 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType))
202 assert config.sourceModel in ["single", "double", "compact", "fit"]
204 # Set the appropriate number of components
205 if config.sourceModel == "single":
206 maxComponents = 1
207 elif config.sourceModel == "double":
208 maxComponents = 2
209 elif config.sourceModel == "compact":
210 maxComponents = 0
211 elif config.sourceModel == "point":
212 raise NotImplementedError("Point source photometry is currently not implemented")
213 elif config.sourceModel == "fit":
214 # It is likely in the future that there will be some heuristic
215 # used to determine what type of model to use for each source,
216 # but that has not yet been implemented (see DM-22551)
217 raise NotImplementedError("sourceModel 'fit' has not been implemented yet")
219 # Convert the centers to pixel coordinates
220 xmin = bbox.getMinX()
221 ymin = bbox.getMinY()
222 centers = [
223 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
224 for peak in footprint.peaks
225 if not isPseudoSource(peak, config.pseudoColumns)
226 ]
228 # Only deblend sources that can be initialized
229 sources, skipped = init_all_sources(
230 frame=frame,
231 centers=centers,
232 observations=observation,
233 thresh=config.morphThresh,
234 max_components=maxComponents,
235 min_snr=config.minSNR,
236 shifting=False,
237 fallback=config.fallback,
238 silent=config.catchFailures,
239 set_spectra=spectrumInit,
240 )
242 # Attach the peak to all of the initialized sources
243 srcIndex = 0
244 for k, center in enumerate(centers):
245 if k not in skipped:
246 # This is just to make sure that there isn't a coding bug
247 assert np.all(sources[srcIndex].center == center)
248 # Store the record for the peak with the appropriate source
249 sources[srcIndex].detectedPeak = footprint.peaks[k]
250 srcIndex += 1
252 # Create the blend and attempt to optimize it
253 blend = Blend(sources, observation)
254 try:
255 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError)
256 except ArithmeticError:
257 # This occurs when a gradient update produces a NaN value
258 # This is usually due to a source initialized with a
259 # negative SED or no flux, often because the peak
260 # is a noise fluctuation in one band and not a real source.
261 iterations = len(blend.loss)
262 failedSources = []
263 for k, src in enumerate(sources):
264 if np.any(~np.isfinite(src.get_model())):
265 failedSources.append(k)
266 raise ScarletGradientError(iterations, failedSources)
268 # Store the location of the PSF center for storage
269 blend.psfCenter = (psfCenter.x, psfCenter.y)
271 return blend, skipped
274def buildLiteObservation(
275 modelPsf,
276 psfCenter,
277 mExposure,
278 footprint=None,
279 badPixelMasks=None,
280 useWeights=True,
281 convolutionType="real",
282):
283 """Generate a LiteObservation from a set of parameters.
285 Make the generation and reconstruction of a scarlet model consistent
286 by building a `LiteObservation` from a set of parameters.
288 Parameters
289 ----------
290 modelPsf : `numpy.ndarray`
291 The 2D model of the PSF in the partially deconvolved space.
292 psfCenter : `tuple` or `Point2I` or `Point2D`
293 The location `(x, y)` used as the center of the PSF.
294 mExposure : `lsst.afw.image.multiband.MultibandExposure`
295 The multi-band exposure that the model represents.
296 If `mExposure` is `None` then no image, variance, or weights are
297 attached to the observation.
298 footprint : `lsst.afw.detection.Footprint`
299 The footprint that is being fit.
300 If `Footprint` is `None` then the weights are not updated to mask
301 out pixels not contained in the footprint.
302 badPixelMasks : `list` of `str`
303 The keys from the bit mask plane used to mask out pixels
304 during the fit.
305 If `badPixelMasks` is `None` then the default values from
306 `ScarletDeblendConfig.badMask` is used.
307 useWeights : `bool`
308 Whether or not fitting should use inverse variance weights to
309 calculate the log-likelihood.
310 convolutionType : `str`
311 The type of convolution to use (either "real" or "fft").
312 When reconstructing an image it is advised to use "real" to avoid
313 polluting the footprint with
315 Returns
316 -------
317 observation : `scarlet.lite.LiteObservation`
318 The observation constructed from the input parameters.
319 """
320 # Initialize the observed PSFs
321 psfModels, mExposure = computePsfKernelImage(mExposure, psfCenter)
323 # Use the inverse variance as the weights
324 if useWeights:
325 weights = 1/mExposure.variance.array
326 else:
327 # Mask out bad pixels
328 weights = np.ones_like(mExposure.image.array)
329 if badPixelMasks is None:
330 badPixelMasks = ScarletDeblendConfig().badMask
331 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks)
332 mask = mExposure.mask.array & badPixels
333 weights[mask > 0] = 0
335 if footprint is not None:
336 # Mask out the pixels outside the footprint
337 weights *= footprint.spans.asArray()
339 observation = lite.LiteObservation(
340 images=mExposure.image.array,
341 variance=mExposure.variance.array,
342 weights=weights,
343 psfs=psfModels,
344 model_psf=modelPsf[None, :, :],
345 convolution_mode=convolutionType,
346 )
348 # Store the bands used to create the observation
349 observation.bands = mExposure.filters
350 return observation
353def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None):
354 """Deblend a parent footprint
356 Parameters
357 ----------
358 mExposure : `lsst.image.MultibandExposure`
359 - The multiband exposure containing the image,
360 mask, and variance data
361 footprint : `lsst.detection.Footprint`
362 - The footprint of the parent to deblend
363 config : `ScarletDeblendConfig`
364 - Configuration of the deblending task
366 Returns
367 -------
368 blend : `scarlet.lite.Blend`
369 The blend this is to be deblended
370 skippedSources : `list[int]`
371 Indices of sources that were skipped due to no flux.
372 This usually means that a source was a spurrious detection in one
373 band that should not have been included in the merged catalog.
374 skippedBands : `list[str]`
375 Bands that were skipped because a PSF could not be generated for them.
376 """
377 # Extract coordinates from each MultiColorPeak
378 bbox = footprint.getBBox()
379 psfCenter = footprint.getCentroid()
381 observation = buildLiteObservation(
382 modelPsf=modelPsf,
383 psfCenter=psfCenter,
384 mExposure=mExposure[:, bbox],
385 footprint=footprint,
386 badPixelMasks=config.badMask,
387 useWeights=config.useWeights,
388 convolutionType=config.convolutionType,
389 )
391 # Convert the centers to pixel coordinates
392 xmin = bbox.getMinX()
393 ymin = bbox.getMinY()
394 centers = [
395 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int)
396 for peak in footprint.peaks
397 if not isPseudoSource(peak, config.pseudoColumns)
398 ]
400 # Initialize the sources
401 if config.morphImage == "chi2":
402 sources = lite.init_all_sources_main(
403 observation,
404 centers,
405 min_snr=config.minSNR,
406 thresh=config.morphThresh,
407 )
408 elif config.morphImage == "wavelet":
409 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin())
410 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)]
411 sources = lite.init_all_sources_wavelets(
412 observation,
413 centers,
414 use_psf=False,
415 wavelets=_wavelets,
416 min_snr=config.minSNR,
417 )
418 else:
419 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.")
421 # Set the optimizer
422 if config.optimizer == "adaprox":
423 parameterization = partial(
424 lite.init_adaprox_component,
425 bg_thresh=config.backgroundThresh,
426 max_prox_iter=config.maxProxIter,
427 )
428 elif config.optimizer == "fista":
429 parameterization = partial(
430 lite.init_fista_component,
431 bg_thresh=config.backgroundThresh,
432 )
433 else:
434 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.")
435 sources = lite.parameterize_sources(sources, observation, parameterization)
437 # Attach the peak to all of the initialized sources
438 for k, center in enumerate(centers):
439 # This is just to make sure that there isn't a coding bug
440 if len(sources[k].components) > 0 and np.any(sources[k].center != center):
441 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}")
442 # Store the record for the peak with the appropriate source
443 sources[k].detectedPeak = footprint.peaks[k]
445 blend = lite.LiteBlend(sources, observation)
447 # Initialize each source with its best fit spectrum
448 if spectrumInit:
449 blend.fit_spectra()
451 # Set the sources that could not be initialized and were skipped
452 skippedSources = [src for src in sources if src.is_null]
454 blend.fit(
455 max_iter=config.maxIter,
456 e_rel=config.relativeError,
457 min_iter=config.minIter,
458 reweight=False,
459 )
461 # Store the location of the PSF center for storage
462 blend.psfCenter = (psfCenter.x, psfCenter.y)
464 # Calculate the bands that were skipped
465 skippedBands = [band for band in mExposure.filters if band not in observation.bands]
467 return blend, skippedSources, skippedBands
470@dataclass
471class DeblenderMetrics:
472 """Metrics and measurements made on single sources.
474 Store deblender metrics to be added as attributes to a scarlet source
475 before it is converted into a `SourceRecord`.
476 When DM-34414 is finished this class will be eliminated and the metrics
477 will be added to the schema using a pipeline task that calculates them
478 from the stored deconvolved models.
480 All of the parameters are one dimensional numpy arrays,
481 with an element for each band in the observed images.
483 `maxOverlap` is useful as a metric for determining how blended a source
484 is because if it only overlaps with other sources at or below
485 the noise level, it is likely to be a mostly isolated source
486 in the deconvolved model frame.
488 `fluxOverlapFraction` is potentially more useful than the canonical
489 "blendedness" (or purity) metric because it accounts for potential
490 biases created during deblending by not weighting the overlapping
491 flux with the flux of this sources model.
493 Attributes
494 ----------
495 maxOverlap : `numpy.ndarray`
496 The maximum overlap that the source has with its neighbors in
497 a single pixel.
498 fluxOverlap : `numpy.ndarray`
499 The total flux from neighbors overlapping with the current source.
500 fluxOverlapFraction : `numpy.ndarray`
501 The fraction of `flux from neighbors/source flux` for a
502 given source within the source's footprint.
503 blendedness : `numpy.ndarray`
504 The metric for determining how blended a source is using the
505 Bosch et al. 2018 metric for "blendedness." Note that some
506 surveys use the term "purity," which is `1-blendedness`.
507 """
508 maxOverlap: np.array
509 fluxOverlap: np.array
510 fluxOverlapFraction: np.array
511 blendedness: np.array
514def setDeblenderMetrics(blend):
515 """Set metrics that can be used to evalute the deblender accuracy
517 This function calculates the `DeblenderMetrics` for each source in the
518 blend, and assigns it to that sources `metrics` property in place.
520 Parameters
521 ----------
522 blend : `scarlet.lite.Blend`
523 The blend containing the sources to measure.
524 """
525 # Store the full model of the scene for comparison
526 blendModel = blend.get_model()
527 for k, src in enumerate(blend.sources):
528 # Extract the source model in the full bounding box
529 model = src.get_model(bbox=blend.bbox)
530 # The footprint is the 2D array of non-zero pixels in each band
531 footprint = np.bitwise_or.reduce(model > 0, axis=0)
532 # Calculate the metrics.
533 # See `DeblenderMetrics` for a description of each metric.
534 neighborOverlap = (blendModel-model) * footprint[None, :, :]
535 maxOverlap = np.max(neighborOverlap, axis=(1, 2))
536 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2))
537 fluxModel = np.sum(model, axis=(1, 2))
538 fluxOverlapFraction = np.zeros((len(model), ), dtype=float)
539 isFinite = fluxModel > 0
540 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite]
541 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2))
542 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness)
545class ScarletDeblendConfig(pexConfig.Config):
546 """MultibandDeblendConfig
548 Configuration for the multiband deblender.
549 The parameters are organized by the parameter types, which are
550 - Stopping Criteria: Used to determine if the fit has converged
551 - Position Fitting Criteria: Used to fit the positions of the peaks
552 - Constraints: Used to apply constraints to the peaks and their components
553 - Other: Parameters that don't fit into the above categories
554 """
555 # Stopping Criteria
556 minIter = pexConfig.Field(dtype=int, default=15,
557 doc="Minimum number of iterations before the optimizer is allowed to stop.")
558 maxIter = pexConfig.Field(dtype=int, default=300,
559 doc=("Maximum number of iterations to deblend a single parent"))
560 relativeError = pexConfig.Field(dtype=float, default=1e-2,
561 doc=("Change in the loss function between iterations to exit fitter. "
562 "Typically this is `1e-2` if measurements will be made on the "
563 "flux re-distributed models and `1e-4` when making measurements "
564 "on the models themselves."))
566 # Constraints
567 morphThresh = pexConfig.Field(dtype=float, default=1,
568 doc="Fraction of background RMS a pixel must have"
569 "to be included in the initial morphology")
570 # Lite Parameters
571 # All of these parameters (except version) are only valid if version='lite'
572 version = pexConfig.ChoiceField(
573 dtype=str,
574 default="lite",
575 allowed={
576 "scarlet": "main scarlet version (likely to be deprecated soon)",
577 "lite": "Optimized version of scarlet for survey data from a single instrument",
578 },
579 doc="The version of scarlet to use.",
580 )
581 optimizer = pexConfig.ChoiceField(
582 dtype=str,
583 default="adaprox",
584 allowed={
585 "adaprox": "Proximal ADAM optimization",
586 "fista": "Accelerated proximal gradient method",
587 },
588 doc="The optimizer to use for fitting parameters and is only used when version='lite'",
589 )
590 morphImage = pexConfig.ChoiceField(
591 dtype=str,
592 default="chi2",
593 allowed={
594 "chi2": "Initialize sources on a chi^2 image made from all available bands",
595 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image",
596 },
597 doc="The type of image to use for initializing the morphology. "
598 "Must be either 'chi2' or 'wavelet'. "
599 )
600 backgroundThresh = pexConfig.Field(
601 dtype=float,
602 default=0.25,
603 doc="Fraction of background to use for a sparsity threshold. "
604 "This prevents sources from growing unrealistically outside "
605 "the parent footprint while still modeling flux correctly "
606 "for bright sources."
607 )
608 maxProxIter = pexConfig.Field(
609 dtype=int,
610 default=1,
611 doc="Maximum number of proximal operator iterations inside of each "
612 "iteration of the optimizer. "
613 "This config field is only used if version='lite' and optimizer='adaprox'."
614 )
615 waveletScales = pexConfig.Field(
616 dtype=int,
617 default=5,
618 doc="Number of wavelet scales to use for wavelet initialization. "
619 "This field is only used when `version`='lite' and `morphImage`='wavelet'."
620 )
622 # Other scarlet paremeters
623 useWeights = pexConfig.Field(
624 dtype=bool, default=True,
625 doc=("Whether or not use use inverse variance weighting."
626 "If `useWeights` is `False` then flat weights are used"))
627 modelPsfSize = pexConfig.Field(
628 dtype=int, default=11,
629 doc="Model PSF side length in pixels")
630 modelPsfSigma = pexConfig.Field(
631 dtype=float, default=0.8,
632 doc="Define sigma for the model frame PSF")
633 minSNR = pexConfig.Field(
634 dtype=float, default=50,
635 doc="Minimum Signal to noise to accept the source."
636 "Sources with lower flux will be initialized with the PSF but updated "
637 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).")
638 saveTemplates = pexConfig.Field(
639 dtype=bool, default=True,
640 doc="Whether or not to save the SEDs and templates")
641 processSingles = pexConfig.Field(
642 dtype=bool, default=True,
643 doc="Whether or not to process isolated sources in the deblender")
644 convolutionType = pexConfig.Field(
645 dtype=str, default="fft",
646 doc="Type of convolution to render the model to the observations.\n"
647 "- 'fft': perform convolutions in Fourier space\n"
648 "- 'real': peform convolutions in real space.")
649 sourceModel = pexConfig.Field(
650 dtype=str, default="double",
651 doc=("How to determine which model to use for sources, from\n"
652 "- 'single': use a single component for all sources\n"
653 "- 'double': use a bulge disk model for all sources\n"
654 "- 'compact': use a single component model, initialzed with a point source morphology, "
655 " for all sources\n"
656 "- 'point': use a point-source model for all sources\n"
657 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"),
658 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.",
659 )
660 setSpectra = pexConfig.Field(
661 dtype=bool, default=True,
662 doc="Whether or not to solve for the best-fit spectra during initialization. "
663 "This makes initialization slightly longer, as it requires a convolution "
664 "to set the optimal spectra, but results in a much better initial log-likelihood "
665 "and reduced total runtime, with convergence in fewer iterations."
666 "This option is only used when "
667 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.")
669 # Mask-plane restrictions
670 badMask = pexConfig.ListField(
671 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"],
672 doc="Whether or not to process isolated sources in the deblender")
673 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
674 doc="Mask planes to ignore when performing statistics")
675 maskLimits = pexConfig.DictField(
676 keytype=str,
677 itemtype=float,
678 default={},
679 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
680 "Sources violating this limit will not be deblended. "
681 "If the fraction is `0` then the limit is a single pixel."),
682 )
684 # Size restrictions
685 maxNumberOfPeaks = pexConfig.Field(
686 dtype=int, default=200,
687 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
688 " (<= 0: unlimited)"))
689 maxFootprintArea = pexConfig.Field(
690 dtype=int, default=100_000,
691 doc=("Maximum area for footprints before they are ignored as large; "
692 "non-positive means no threshold applied"))
693 maxAreaTimesPeaks = pexConfig.Field(
694 dtype=int, default=10_000_000,
695 doc=("Maximum rectangular footprint area * nPeaks in the footprint. "
696 "This was introduced in DM-33690 to prevent fields that are crowded or have a "
697 "LSB galaxy that causes memory intensive initialization in scarlet from dominating "
698 "the overall runtime and/or causing the task to run out of memory. "
699 "(<= 0: unlimited)")
700 )
701 maxFootprintSize = pexConfig.Field(
702 dtype=int, default=0,
703 doc=("Maximum linear dimension for footprints before they are ignored "
704 "as large; non-positive means no threshold applied"))
705 minFootprintAxisRatio = pexConfig.Field(
706 dtype=float, default=0.0,
707 doc=("Minimum axis ratio for footprints before they are ignored "
708 "as large; non-positive means no threshold applied"))
709 maxSpectrumCutoff = pexConfig.Field(
710 dtype=int, default=1_000_000,
711 doc=("Maximum number of pixels * number of sources in a blend. "
712 "This is different than `maxFootprintArea` because this isn't "
713 "the footprint area but the area of the bounding box that "
714 "contains the footprint, and is also multiplied by the number of"
715 "sources in the footprint. This prevents large skinny blends with "
716 "a high density of sources from running out of memory. "
717 "If `maxSpectrumCutoff == -1` then there is no cutoff.")
718 )
719 # Failure modes
720 fallback = pexConfig.Field(
721 dtype=bool, default=True,
722 doc="Whether or not to fallback to a smaller number of components if a source does not initialize"
723 )
724 notDeblendedMask = pexConfig.Field(
725 dtype=str, default="NOT_DEBLENDED", optional=True,
726 doc="Mask name for footprints not deblended, or None")
727 catchFailures = pexConfig.Field(
728 dtype=bool, default=True,
729 doc=("If True, catch exceptions thrown by the deblender, log them, "
730 "and set a flag on the parent, instead of letting them propagate up"))
732 # Other options
733 columnInheritance = pexConfig.DictField(
734 keytype=str, itemtype=str, default={
735 "deblend_nChild": "deblend_parentNChild",
736 "deblend_nPeaks": "deblend_parentNPeaks",
737 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag",
738 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag",
739 },
740 doc="Columns to pass from the parent to the child. "
741 "The key is the name of the column for the parent record, "
742 "the value is the name of the column to use for the child."
743 )
744 pseudoColumns = pexConfig.ListField(
745 dtype=str, default=['merge_peak_sky', 'sky_source'],
746 doc="Names of flags which should never be deblended."
747 )
749 # Testing options
750 # Some obs packages and ci packages run the full pipeline on a small
751 # subset of data to test that the pipeline is functioning properly.
752 # This is not meant as scientific validation, so it can be useful
753 # to only run on a small subset of the data that is large enough to
754 # test the desired pipeline features but not so long that the deblender
755 # is the tall pole in terms of execution times.
756 useCiLimits = pexConfig.Field(
757 dtype=bool, default=False,
758 doc="Limit the number of sources deblended for CI to prevent long build times")
759 ciDeblendChildRange = pexConfig.ListField(
760 dtype=int, default=[5, 10],
761 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
762 "If `useCiLimits==False` then this parameter is ignored.")
763 ciNumParentsToDeblend = pexConfig.Field(
764 dtype=int, default=10,
765 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
766 "within `ciDebledChildRange`. "
767 "If `useCiLimits==False` then this parameter is ignored.")
770class ScarletDeblendTask(pipeBase.Task):
771 """ScarletDeblendTask
773 Split blended sources into individual sources.
775 This task has no return value; it only modifies the SourceCatalog in-place.
776 """
777 ConfigClass = ScarletDeblendConfig
778 _DefaultName = "scarletDeblend"
780 def __init__(self, schema, peakSchema=None, **kwargs):
781 """Create the task, adding necessary fields to the given schema.
783 Parameters
784 ----------
785 schema : `lsst.afw.table.schema.schema.Schema`
786 Schema object for measurement fields; will be modified in-place.
787 peakSchema : `lsst.afw.table.schema.schema.Schema`
788 Schema of Footprint Peaks that will be passed to the deblender.
789 Any fields beyond the PeakTable minimal schema will be transferred
790 to the main source Schema. If None, no fields will be transferred
791 from the Peaks.
792 filters : list of str
793 Names of the filters used for the eposures. This is needed to store
794 the SED as a field
795 **kwargs
796 Passed to Task.__init__.
797 """
798 pipeBase.Task.__init__(self, **kwargs)
800 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
801 if peakSchema is None:
802 # In this case, the peakSchemaMapper will transfer nothing, but
803 # we'll still have one
804 # to simplify downstream code
805 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
806 else:
807 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
808 for item in peakSchema:
809 if item.key not in peakMinimalSchema:
810 self.peakSchemaMapper.addMapping(item.key, item.field)
811 # Because SchemaMapper makes a copy of the output schema
812 # you give its ctor, it isn't updating this Schema in
813 # place. That's probably a design flaw, but in the
814 # meantime, we'll keep that schema in sync with the
815 # peakSchemaMapper.getOutputSchema() manually, by adding
816 # the same fields to both.
817 schema.addField(item.field)
818 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
819 self._addSchemaKeys(schema)
820 self.schema = schema
821 self.toCopyFromParent = [item.key for item in self.schema
822 if item.field.getName().startswith("merge_footprint")]
824 def _addSchemaKeys(self, schema):
825 """Add deblender specific keys to the schema
826 """
827 # Parent (blend) fields
828 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms')
829 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge')
830 self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
831 doc='Number of children this object has (defaults to 0)')
832 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32,
833 doc="Number of initial peaks in the blend. "
834 "This includes peaks that may have been culled "
835 "during deblending or failed to deblend")
836 # Skipped flags
837 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
838 doc="Deblender skipped this source")
839 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag',
840 doc='The source has only a single peak '
841 'and was not deblended')
842 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag',
843 doc='The source is identified as a "pseudo" source and '
844 'was not deblended')
845 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
846 doc='Source had too many peaks; '
847 'only the brightest were included')
848 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
849 doc='Parent footprint covered too many pixels')
850 self.maskedKey = schema.addField('deblend_masked', type='Flag',
851 doc='Parent footprint had too many masked pixels')
852 # Convergence flags
853 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag',
854 doc='scarlet sed optimization did not converge before'
855 'config.maxIter')
856 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag',
857 doc='scarlet morph optimization did not converge before'
858 'config.maxIter')
859 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag',
860 type='Flag',
861 doc='at least one source in the blend'
862 'failed to converge')
863 # Error flags
864 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
865 doc="Deblending failed on source")
866 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25,
867 doc='Name of error if the blend failed')
868 self.incompleteDataKey = schema.addField('deblend_incompleteData', type='Flag',
869 doc='True when a blend has at least one band '
870 'that could not generate a PSF and was '
871 'not included in the model.')
872 # Deblended source fields
873 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center",
874 doc="Center used to apply constraints in scarlet",
875 unit="pixel")
876 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32,
877 doc="ID of the peak in the parent footprint. "
878 "This is not unique, but the combination of 'parent'"
879 "and 'peakId' should be for all child sources. "
880 "Top level blends with no parents have 'peakId=0'")
881 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count',
882 doc="The instFlux at the peak position of deblended mode")
883 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25,
884 doc="The type of model used, for example "
885 "MultiExtendedSource, SingleExtendedSource, PointSource")
886 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32,
887 doc="deblend_nPeaks from this records parent.")
888 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32,
889 doc="deblend_nChild from this records parent.")
890 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32,
891 doc="Flux measurement from scarlet")
892 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32,
893 doc="Final logL, used to identify regressions in scarlet.")
894 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag',
895 doc='Source had flux on the edge of the parent footprint')
896 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag',
897 doc="True when scarlet initializes sources "
898 "in the blend with a more accurate spectrum. "
899 "The algorithm uses a lot of memory, "
900 "so large dense blends will use "
901 "a less accurate initialization.")
902 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32,
903 doc="Number of components in a ScarletLiteSource. "
904 "If `config.version != 'lite'`then "
905 "this column is set to zero.")
906 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
907 doc='Deblender thought this source looked like a PSF')
908 self.coverageKey = schema.addField('deblend_dataCoverage', type=np.float32,
909 doc='Fraction of pixels with data. '
910 'In other words, 1 - fraction of pixels with NO_DATA set.')
911 # Blendedness/classification metrics
912 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32,
913 doc="Maximum overlap with all of the other neighbors flux "
914 "combined."
915 "This is useful as a metric for determining how blended a "
916 "source is because if it only overlaps with other sources "
917 "at or below the noise level, it is likely to be a mostly "
918 "isolated source in the deconvolved model frame.")
919 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32,
920 doc="This is the total flux from neighboring objects that "
921 "overlaps with this source.")
922 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32,
923 doc="This is the fraction of "
924 "`flux from neighbors/source flux` "
925 "for a given source within the source's"
926 "footprint.")
927 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32,
928 doc="The Bosch et al. 2018 metric for 'blendedness.' ")
930 @timeMethod
931 def run(self, mExposure, mergedSources):
932 """Get the psf from each exposure and then run deblend().
934 Parameters
935 ----------
936 mExposure : `MultibandExposure`
937 The exposures should be co-added images of the same
938 shape and region of the sky.
939 mergedSources : `SourceCatalog`
940 The merged `SourceCatalog` that contains parent footprints
941 to (potentially) deblend.
943 Returns
944 -------
945 templateCatalogs: dict
946 Keys are the names of the filters and the values are
947 `lsst.afw.table.source.source.SourceCatalog`'s.
948 These are catalogs with heavy footprints that are the templates
949 created by the multiband templates.
950 """
951 return self.deblend(mExposure, mergedSources)
953 @timeMethod
954 def deblend(self, mExposure, catalog):
955 """Deblend a data cube of multiband images
957 Parameters
958 ----------
959 mExposure : `MultibandExposure`
960 The exposures should be co-added images of the same
961 shape and region of the sky.
962 catalog : `SourceCatalog`
963 The merged `SourceCatalog` that contains parent footprints
964 to (potentially) deblend. The new deblended sources are
965 appended to this catalog in place.
967 Returns
968 -------
969 catalogs : `dict` or `None`
970 Keys are the names of the filters and the values are
971 `lsst.afw.table.source.source.SourceCatalog`'s.
972 These are catalogs with heavy footprints that are the templates
973 created by the multiband templates.
974 """
975 import time
977 # Cull footprints if required by ci
978 if self.config.useCiLimits:
979 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.",
980 len(catalog))
981 # Select parents with a number of children in the range
982 # config.ciDeblendChildRange
983 minChildren, maxChildren = self.config.ciDeblendChildRange
984 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog])
985 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
986 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
987 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range "
988 "indicated by ciDeblendChildRange. Adjust this range to include more "
989 "parents.")
990 # Keep all of the isolated parents and the first
991 # `ciNumParentsToDeblend` children
992 parents = nPeaks == 1
993 children = np.zeros((len(catalog),), dtype=bool)
994 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True
995 catalog = catalog[parents | children]
996 # We need to update the IdFactory, otherwise the the source ids
997 # will not be sequential
998 idFactory = catalog.getIdFactory()
999 maxId = np.max(catalog["id"])
1000 idFactory.notify(maxId)
1002 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure))
1003 periodicLog = PeriodicLogger(self.log)
1005 # Create a set of wavelet coefficients if using wavelet initialization
1006 if self.config.version == "lite" and self.config.morphImage == "wavelet":
1007 images = mExposure.image.array
1008 variance = mExposure.variance.array
1009 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales)
1010 else:
1011 wavelets = None
1013 # Add the NOT_DEBLENDED mask to the mask plane in each band
1014 if self.config.notDeblendedMask:
1015 for mask in mExposure.mask:
1016 mask.addMaskPlane(self.config.notDeblendedMask)
1018 # Initialize the persistable data model
1019 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma)
1020 dataModel = ScarletModelData(modelPsf)
1022 nParents = len(catalog)
1023 nDeblendedParents = 0
1024 skippedParents = []
1025 for parentIndex in range(nParents):
1026 parent = catalog[parentIndex]
1027 foot = parent.getFootprint()
1028 bbox = foot.getBBox()
1029 peaks = foot.getPeaks()
1031 # Since we use the first peak for the parent object, we should
1032 # propagate its flags to the parent source.
1033 parent.assign(peaks[0], self.peakSchemaMapper)
1035 # Block of conditions for skipping a parent with multiple children
1036 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None:
1037 self._skipParent(parent, *skipArgs)
1038 skippedParents.append(parentIndex)
1039 continue
1041 nDeblendedParents += 1
1042 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks))
1043 # Run the deblender
1044 blendError = None
1046 # Choose whether or not to use improved spectral initialization.
1047 # This significantly cuts down on the number of iterations
1048 # that the optimizer needs and usually results in a better
1049 # fit.
1050 # But using least squares on a very large blend causes memory
1051 # issues, so it is not done for large blends
1052 if self.config.setSpectra:
1053 if self.config.maxSpectrumCutoff <= 0:
1054 spectrumInit = True
1055 else:
1056 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff
1057 else:
1058 spectrumInit = False
1060 try:
1061 t0 = time.monotonic()
1062 # Build the parameter lists with the same ordering
1063 if self.config.version == "scarlet":
1064 blend, skippedSources = deblend(mExposure, foot, self.config, spectrumInit)
1065 skippedBands = []
1066 elif self.config.version == "lite":
1067 blend, skippedSources, skippedBands = deblend_lite(
1068 mExposure=mExposure,
1069 modelPsf=modelPsf,
1070 footprint=foot,
1071 config=self.config,
1072 spectrumInit=spectrumInit,
1073 wavelets=wavelets,
1074 )
1075 tf = time.monotonic()
1076 runtime = (tf-t0)*1000
1077 converged = _checkBlendConvergence(blend, self.config.relativeError)
1078 # Store the number of components in the blend
1079 if self.config.version == "lite":
1080 nComponents = len(blend.components)
1081 else:
1082 nComponents = 0
1083 nChild = len(blend.sources)
1084 parent.set(self.incompleteDataKey, len(skippedBands) > 0)
1085 # Catch all errors and filter out the ones that we know about
1086 except Exception as e:
1087 blendError = type(e).__name__
1088 if isinstance(e, ScarletGradientError):
1089 parent.set(self.iterKey, e.iterations)
1090 else:
1091 blendError = "UnknownError"
1092 if self.config.catchFailures:
1093 # Make it easy to find UnknownErrors in the log file
1094 self.log.warn("UnknownError")
1095 import traceback
1096 traceback.print_exc()
1097 else:
1098 raise
1100 self._skipParent(
1101 parent=parent,
1102 skipKey=self.deblendFailedKey,
1103 logMessage=f"Unable to deblend source {parent.getId}: {blendError}",
1104 )
1105 parent.set(self.deblendErrorKey, blendError)
1106 skippedParents.append(parentIndex)
1107 continue
1109 # Update the parent record with the deblending results
1110 if self.config.version == "scarlet":
1111 logL = -blend.loss[-1] + blend.observations[0].log_norm
1112 elif self.config.version == "lite":
1113 logL = blend.loss[-1]
1114 self._updateParentRecord(
1115 parent=parent,
1116 nPeaks=len(peaks),
1117 nChild=nChild,
1118 nComponents=nComponents,
1119 runtime=runtime,
1120 iterations=len(blend.loss),
1121 logL=logL,
1122 spectrumInit=spectrumInit,
1123 converged=converged,
1124 )
1126 # Add each deblended source to the catalog
1127 for k, scarletSource in enumerate(blend.sources):
1128 # Skip any sources with no flux or that scarlet skipped because
1129 # it could not initialize
1130 if k in skippedSources or (self.config.version == "lite" and scarletSource.is_null):
1131 # No need to propagate anything
1132 continue
1133 parent.set(self.deblendSkippedKey, False)
1135 # Add all fields except the HeavyFootprint to the
1136 # source record
1137 sourceRecord = self._addChild(
1138 parent=parent,
1139 peak=scarletSource.detectedPeak,
1140 catalog=catalog,
1141 scarletSource=scarletSource,
1142 )
1143 scarletSource.recordId = sourceRecord.getId()
1144 scarletSource.peakId = scarletSource.detectedPeak.getId()
1146 # Store the blend information so that it can be persisted
1147 if self.config.version == "lite":
1148 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin(), blend.observation.bands)
1149 else:
1150 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin(), mExposure.filters)
1151 dataModel.blends[parent.getId()] = blendData
1153 # Log a message if it has been a while since the last log.
1154 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents)
1156 # Clear the cached values in scarlet to clear out memory
1157 scarlet.cache.Cache._cache = {}
1159 # Update the mExposure mask with the footprint of skipped parents
1160 if self.config.notDeblendedMask:
1161 for mask in mExposure.mask:
1162 for parentIndex in skippedParents:
1163 fp = catalog[parentIndex].getFootprint()
1164 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
1166 self.log.info("Deblender results: of %d parent sources, %d were deblended, "
1167 "creating %d children, for a total of %d sources",
1168 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog))
1169 return catalog, dataModel
1171 def _isLargeFootprint(self, footprint):
1172 """Returns whether a Footprint is large
1174 'Large' is defined by thresholds on the area, size and axis ratio,
1175 and total area of the bounding box multiplied by
1176 the number of children.
1177 These may be disabled independently by configuring them to be
1178 non-positive.
1179 """
1180 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
1181 return True
1182 if self.config.maxFootprintSize > 0:
1183 bbox = footprint.getBBox()
1184 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
1185 return True
1186 if self.config.minFootprintAxisRatio > 0:
1187 axes = afwEll.Axes(footprint.getShape())
1188 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
1189 return True
1190 if self.config.maxAreaTimesPeaks > 0:
1191 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks:
1192 return True
1193 return False
1195 def _isMasked(self, footprint, mExposure):
1196 """Returns whether the footprint violates the mask limits
1198 Parameters
1199 ----------
1200 footprint : `lsst.afw.detection.Footprint`
1201 The footprint to check for masked pixels
1202 mMask : `lsst.afw.image.MaskX`
1203 The mask plane to check for masked pixels in the `footprint`.
1205 Returns
1206 -------
1207 isMasked : `bool`
1208 `True` if `self.config.maskPlaneLimits` is less than the
1209 fraction of pixels for a given mask in
1210 `self.config.maskLimits`.
1211 """
1212 bbox = footprint.getBBox()
1213 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0)
1214 size = float(footprint.getArea())
1215 for maskName, limit in self.config.maskLimits.items():
1216 maskVal = mExposure.mask.getPlaneBitMask(maskName)
1217 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin())
1218 # spanset of masked pixels
1219 maskedSpan = footprint.spans.intersect(_mask, maskVal)
1220 if (maskedSpan.getArea())/size > limit:
1221 return True
1222 return False
1224 def _skipParent(self, parent, skipKey, logMessage):
1225 """Update a parent record that is not being deblended.
1227 This is a fairly trivial function but is implemented to ensure
1228 that a skipped parent updates the appropriate columns
1229 consistently, and always has a flag to mark the reason that
1230 it is being skipped.
1232 Parameters
1233 ----------
1234 parent : `lsst.afw.table.source.source.SourceRecord`
1235 The parent record to flag as skipped.
1236 skipKey : `bool`
1237 The name of the flag to mark the reason for skipping.
1238 logMessage : `str`
1239 The message to display in a log.trace when a source
1240 is skipped.
1241 """
1242 if logMessage is not None:
1243 self.log.trace(logMessage)
1244 self._updateParentRecord(
1245 parent=parent,
1246 nPeaks=len(parent.getFootprint().peaks),
1247 nChild=0,
1248 nComponents=0,
1249 runtime=np.nan,
1250 iterations=0,
1251 logL=np.nan,
1252 spectrumInit=False,
1253 converged=False,
1254 )
1256 # Mark the source as skipped by the deblender and
1257 # flag the reason why.
1258 parent.set(self.deblendSkippedKey, True)
1259 parent.set(skipKey, True)
1261 def _checkSkipped(self, parent, mExposure):
1262 """Update a parent record that is not being deblended.
1264 This is a fairly trivial function but is implemented to ensure
1265 that a skipped parent updates the appropriate columns
1266 consistently, and always has a flag to mark the reason that
1267 it is being skipped.
1269 Parameters
1270 ----------
1271 parent : `lsst.afw.table.source.source.SourceRecord`
1272 The parent record to flag as skipped.
1273 mExposure : `MultibandExposure`
1274 The exposures should be co-added images of the same
1275 shape and region of the sky.
1276 Returns
1277 -------
1278 skip: `bool`
1279 `True` if the deblender will skip the parent
1280 """
1281 skipKey = None
1282 skipMessage = None
1283 footprint = parent.getFootprint()
1284 if len(footprint.peaks) < 2 and not self.config.processSingles:
1285 # Skip isolated sources unless processSingles is turned on.
1286 # Note: this does not flag isolated sources as skipped or
1287 # set the NOT_DEBLENDED mask in the exposure,
1288 # since these aren't really any skipped blends.
1289 skipKey = self.isolatedParentKey
1290 elif isPseudoSource(parent, self.config.pseudoColumns):
1291 # We also skip pseudo sources, like sky objects, which
1292 # are intended to be skipped.
1293 skipKey = self.pseudoKey
1294 if self._isLargeFootprint(footprint):
1295 # The footprint is above the maximum footprint size limit
1296 skipKey = self.tooBigKey
1297 skipMessage = f"Parent {parent.getId()}: skipping large footprint"
1298 elif self._isMasked(footprint, mExposure):
1299 # The footprint exceeds the maximum number of masked pixels
1300 skipKey = self.maskedKey
1301 skipMessage = f"Parent {parent.getId()}: skipping masked footprint"
1302 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks:
1303 # Unlike meas_deblender, in scarlet we skip the entire blend
1304 # if the number of peaks exceeds max peaks, since neglecting
1305 # to model any peaks often results in catastrophic failure
1306 # of scarlet to generate models for the brighter sources.
1307 skipKey = self.tooManyPeaksKey
1308 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks"
1309 if skipKey is not None:
1310 return (skipKey, skipMessage)
1311 return None
1313 def setSkipFlags(self, mExposure, catalog):
1314 """Set the skip flags for all of the parent sources
1316 This is mostly used for testing which parent sources will be deblended
1317 and which will be skipped based on the current configuration options.
1318 Skipped sources will have the appropriate flags set in place in the
1319 catalog.
1321 Parameters
1322 ----------
1323 mExposure : `MultibandExposure`
1324 The exposures should be co-added images of the same
1325 shape and region of the sky.
1326 catalog : `SourceCatalog`
1327 The merged `SourceCatalog` that contains parent footprints
1328 to (potentially) deblend. The new deblended sources are
1329 appended to this catalog in place.
1330 """
1331 for src in catalog:
1332 if skipArgs := self._checkSkipped(src, mExposure) is not None:
1333 self._skipParent(src, *skipArgs)
1335 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents,
1336 runtime, iterations, logL, spectrumInit, converged):
1337 """Update a parent record in all of the single band catalogs.
1339 Ensure that all locations that update a parent record,
1340 whether it is skipped or updated after deblending,
1341 update all of the appropriate columns.
1343 Parameters
1344 ----------
1345 parent : `lsst.afw.table.source.source.SourceRecord`
1346 The parent record to update.
1347 nPeaks : `int`
1348 Number of peaks in the parent footprint.
1349 nChild : `int`
1350 Number of children deblended from the parent.
1351 This may differ from `nPeaks` if some of the peaks
1352 were culled and have no deblended model.
1353 nComponents : `int`
1354 Total number of components in the parent.
1355 This is usually different than the number of children,
1356 since it is common for a single source to have multiple
1357 components.
1358 runtime : `float`
1359 Total runtime for deblending.
1360 iterations : `int`
1361 Total number of iterations in scarlet before convergence.
1362 logL : `float`
1363 Final log likelihood of the blend.
1364 spectrumInit : `bool`
1365 True when scarlet used `set_spectra` to initialize all
1366 sources with better initial intensities.
1367 converged : `bool`
1368 True when the optimizer reached convergence before
1369 reaching the maximum number of iterations.
1370 """
1371 parent.set(self.nPeaksKey, nPeaks)
1372 parent.set(self.nChildKey, nChild)
1373 parent.set(self.nComponentsKey, nComponents)
1374 parent.set(self.runtimeKey, runtime)
1375 parent.set(self.iterKey, iterations)
1376 parent.set(self.scarletLogLKey, logL)
1377 parent.set(self.scarletSpectrumInitKey, spectrumInit)
1378 parent.set(self.blendConvergenceFailedFlagKey, converged)
1380 def _addChild(self, parent, peak, catalog, scarletSource):
1381 """Add a child to a catalog.
1383 This creates a new child in the source catalog,
1384 assigning it a parent id, and adding all columns
1385 that are independent across all filter bands.
1387 Parameters
1388 ----------
1389 parent : `lsst.afw.table.source.source.SourceRecord`
1390 The parent of the new child record.
1391 peak : `lsst.afw.table.PeakRecord`
1392 The peak record for the peak from the parent peak catalog.
1393 catalog : `lsst.afw.table.source.source.SourceCatalog`
1394 The merged `SourceCatalog` that contains parent footprints
1395 to (potentially) deblend.
1396 scarletSource : `scarlet.Component`
1397 The scarlet model for the new source record.
1398 """
1399 src = catalog.addNew()
1400 for key in self.toCopyFromParent:
1401 src.set(key, parent.get(key))
1402 # The peak catalog is the same for all bands,
1403 # so we just use the first peak catalog
1404 src.assign(peak, self.peakSchemaMapper)
1405 src.setParent(parent.getId())
1406 src.set(self.nPeaksKey, 1)
1407 # Set the psf key based on whether or not the source was
1408 # deblended using the PointSource model.
1409 # This key is not that useful anymore since we now keep track of
1410 # `modelType`, but we continue to propagate it in case code downstream
1411 # is expecting it.
1412 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource")
1413 src.set(self.modelTypeKey, scarletSource.__class__.__name__)
1414 # We set the runtime to zero so that summing up the
1415 # runtime column will give the total time spent
1416 # running the deblender for the catalog.
1417 src.set(self.runtimeKey, 0)
1419 # Set the position of the peak from the parent footprint
1420 # This will make it easier to match the same source across
1421 # deblenders and across observations, where the peak
1422 # position is unlikely to change unless enough time passes
1423 # for a source to move on the sky.
1424 src.set(self.peakCenter, geom.Point2I(peak["i_x"], peak["i_y"]))
1425 src.set(self.peakIdKey, peak["id"])
1427 # Store the number of components for the source
1428 src.set(self.nComponentsKey, len(scarletSource.components))
1430 # Flag sources missing one or more bands
1431 src.set(self.incompleteDataKey, parent.get(self.incompleteDataKey))
1433 # Propagate columns from the parent to the child
1434 for parentColumn, childColumn in self.config.columnInheritance.items():
1435 src.set(childColumn, parent.get(parentColumn))
1437 return src