Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%

444 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-12 03:02 -0800

1# This file is part of meas_extensions_scarlet. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from dataclasses import dataclass 

23from functools import partial 

24import logging 

25import numpy as np 

26import scarlet 

27from scarlet.psf import ImagePSF, GaussianPSF 

28from scarlet import Blend, Frame, Observation 

29from scarlet.renderer import ConvolutionRenderer 

30from scarlet.detect import get_detect_wavelets 

31from scarlet.initialization import init_all_sources 

32from scarlet import lite 

33 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36from lsst.geom import Point2I, Point2D 

37import lsst.afw.geom.ellipses as afwEll 

38import lsst.afw.image as afwImage 

39import lsst.afw.detection as afwDet 

40import lsst.afw.table as afwTable 

41from lsst.utils.logging import PeriodicLogger 

42from lsst.utils.timer import timeMethod 

43from lsst.afw.image.exposure import IncompleteDataError 

44 

45from .source import bboxToScarletBox 

46from .io import ScarletModelData, scarletToData, scarletLiteToData 

47 

48# Scarlet and proxmin have a different definition of log levels than the stack, 

49# so even "warnings" occur far more often than we would like. 

50# So for now we only display scarlet and proxmin errors, as all other 

51# scarlet outputs would be considered "TRACE" by our standards. 

52scarletLogger = logging.getLogger("scarlet") 

53scarletLogger.setLevel(logging.ERROR) 

54proxminLogger = logging.getLogger("proxmin") 

55proxminLogger.setLevel(logging.ERROR) 

56 

57__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"] 

58 

59logger = logging.getLogger(__name__) 

60 

61 

62class ScarletGradientError(Exception): 

63 """An error occurred during optimization 

64 

65 This error occurs when the optimizer encounters 

66 a NaN value while calculating the gradient. 

67 """ 

68 def __init__(self, iterations, sources): 

69 self.iterations = iterations 

70 self.sources = sources 

71 msg = ("ScalarGradientError in iteration {0}. " 

72 "NaN values introduced in sources {1}") 

73 self.message = msg.format(iterations, sources) 

74 

75 def __str__(self): 

76 return self.message 

77 

78 

79def _checkBlendConvergence(blend, f_rel): 

80 """Check whether or not a blend has converged 

81 """ 

82 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1]) 

83 convergence = f_rel * np.abs(blend.loss[-1]) 

84 return deltaLoss < convergence 

85 

86 

87def isPseudoSource(source, pseudoColumns): 

88 """Check if a source is a pseudo source. 

89 

90 This is mostly for skipping sky objects, 

91 but any other column can also be added to disable 

92 deblending on a parent or individual source when 

93 set to `True`. 

94 

95 Parameters 

96 ---------- 

97 source : `lsst.afw.table.source.source.SourceRecord` 

98 The source to check for the pseudo bit. 

99 pseudoColumns : `list` of `str` 

100 A list of columns to check for pseudo sources. 

101 """ 

102 isPseudo = False 

103 for col in pseudoColumns: 

104 try: 

105 isPseudo |= source[col] 

106 except KeyError: 

107 pass 

108 return isPseudo 

109 

110 

111def deblend(mExposure, footprint, config, spectrumInit): 

112 """Deblend a parent footprint 

113 

114 Parameters 

115 ---------- 

116 mExposure : `lsst.image.MultibandExposure` 

117 The multiband exposure containing the image, 

118 mask, and variance data. 

119 footprint : `lsst.detection.Footprint` 

120 The footprint of the parent to deblend. 

121 config : `ScarletDeblendConfig` 

122 Configuration of the deblending task. 

123 spectrumInit : `bool` 

124 Whether or not to initialize the spectrum. 

125 

126 Returns 

127 ------- 

128 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData` 

129 The persistable representation of a `scarlet.Blend`. 

130 skipped : `list` of `int` 

131 The indices of any children that failed to initialize 

132 and were skipped. 

133 spectrumInit : `bool` 

134 Whether or not all of the sources were initialized by jointly 

135 fitting their SED's. This provides a better initialization 

136 but created memory issues when a blend is too large or 

137 contains too many sources. 

138 """ 

139 # Extract coordinates from each MultiColorPeak 

140 bbox = footprint.getBBox() 

141 

142 # Create the data array from the masked images 

143 images = mExposure.image[:, bbox].array 

144 

145 # Use the inverse variance as the weights 

146 if config.useWeights: 

147 weights = 1/mExposure.variance[:, bbox].array 

148 else: 

149 weights = np.ones_like(images) 

150 badPixels = mExposure.mask.getPlaneBitMask(config.badMask) 

151 mask = mExposure.mask[:, bbox].array & badPixels 

152 weights[mask > 0] = 0 

153 

154 # Mask out the pixels outside the footprint 

155 weights *= footprint.spans.asArray() 

156 

157 psfCenter = footprint.getCentroid() 

158 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32) 

159 psfs = ImagePSF(psfs) 

160 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters)) 

161 

162 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters) 

163 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters) 

164 if config.convolutionType == "fft": 

165 observation.match(frame) 

166 elif config.convolutionType == "real": 

167 renderer = ConvolutionRenderer(observation, frame, convolution_type="real") 

168 observation.match(frame, renderer=renderer) 

169 else: 

170 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType)) 

171 

172 assert(config.sourceModel in ["single", "double", "compact", "fit"]) 

173 

174 # Set the appropriate number of components 

175 if config.sourceModel == "single": 

176 maxComponents = 1 

177 elif config.sourceModel == "double": 

178 maxComponents = 2 

179 elif config.sourceModel == "compact": 

180 maxComponents = 0 

181 elif config.sourceModel == "point": 

182 raise NotImplementedError("Point source photometry is currently not implemented") 

183 elif config.sourceModel == "fit": 

184 # It is likely in the future that there will be some heuristic 

185 # used to determine what type of model to use for each source, 

186 # but that has not yet been implemented (see DM-22551) 

187 raise NotImplementedError("sourceModel 'fit' has not been implemented yet") 

188 

189 # Convert the centers to pixel coordinates 

190 xmin = bbox.getMinX() 

191 ymin = bbox.getMinY() 

192 centers = [ 

193 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

194 for peak in footprint.peaks 

195 if not isPseudoSource(peak, config.pseudoColumns) 

196 ] 

197 

198 # Only deblend sources that can be initialized 

199 sources, skipped = init_all_sources( 

200 frame=frame, 

201 centers=centers, 

202 observations=observation, 

203 thresh=config.morphThresh, 

204 max_components=maxComponents, 

205 min_snr=config.minSNR, 

206 shifting=False, 

207 fallback=config.fallback, 

208 silent=config.catchFailures, 

209 set_spectra=spectrumInit, 

210 ) 

211 

212 # Attach the peak to all of the initialized sources 

213 srcIndex = 0 

214 for k, center in enumerate(centers): 

215 if k not in skipped: 

216 # This is just to make sure that there isn't a coding bug 

217 assert np.all(sources[srcIndex].center == center) 

218 # Store the record for the peak with the appropriate source 

219 sources[srcIndex].detectedPeak = footprint.peaks[k] 

220 srcIndex += 1 

221 

222 # Create the blend and attempt to optimize it 

223 blend = Blend(sources, observation) 

224 try: 

225 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError) 

226 except ArithmeticError: 

227 # This occurs when a gradient update produces a NaN value 

228 # This is usually due to a source initialized with a 

229 # negative SED or no flux, often because the peak 

230 # is a noise fluctuation in one band and not a real source. 

231 iterations = len(blend.loss) 

232 failedSources = [] 

233 for k, src in enumerate(sources): 

234 if np.any(~np.isfinite(src.get_model())): 

235 failedSources.append(k) 

236 raise ScarletGradientError(iterations, failedSources) 

237 

238 # Store the location of the PSF center for storage 

239 blend.psfCenter = (psfCenter.x, psfCenter.y) 

240 

241 return blend, skipped 

242 

243 

244def buildLiteObservation( 

245 modelPsf, 

246 psfCenter, 

247 mExposure, 

248 footprint=None, 

249 badPixelMasks=None, 

250 useWeights=True, 

251 convolutionType="real", 

252): 

253 """Generate a LiteObservation from a set of parameters. 

254 

255 Make the generation and reconstruction of a scarlet model consistent 

256 by building a `LiteObservation` from a set of parameters. 

257 

258 Parameters 

259 ---------- 

260 modelPsf : `numpy.ndarray` 

261 The 2D model of the PSF in the partially deconvolved space. 

262 psfCenter : `tuple` or `Point2I` or `Point2D` 

263 The location `(x, y)` used as the center of the PSF. 

264 mExposure : `lsst.afw.image.multiband.MultibandExposure` 

265 The multi-band exposure that the model represents. 

266 If `mExposure` is `None` then no image, variance, or weights are 

267 attached to the observation. 

268 footprint : `lsst.afw.detection.Footprint` 

269 The footprint that is being fit. 

270 If `Footprint` is `None` then the weights are not updated to mask 

271 out pixels not contained in the footprint. 

272 badPixelMasks : `list` of `str` 

273 The keys from the bit mask plane used to mask out pixels 

274 during the fit. 

275 If `badPixelMasks` is `None` then the default values from 

276 `ScarletDeblendConfig.badMask` is used. 

277 useWeights : `bool` 

278 Whether or not fitting should use inverse variance weights to 

279 calculate the log-likelihood. 

280 convolutionType : `str` 

281 The type of convolution to use (either "real" or "fft"). 

282 When reconstructing an image it is advised to use "real" to avoid 

283 polluting the footprint with 

284 

285 Returns 

286 ------- 

287 observation : `scarlet.lite.LiteObservation` 

288 The observation constructed from the input parameters. 

289 """ 

290 # Initialize the observed PSFs 

291 if not isinstance(psfCenter, Point2D): 

292 psfCenter = Point2D(*psfCenter) 

293 psfModels = mExposure.computePsfKernelImage(psfCenter) 

294 

295 # Use the inverse variance as the weights 

296 if useWeights: 

297 weights = 1/mExposure.variance.array 

298 else: 

299 # Mask out bad pixels 

300 weights = np.ones_like(mExposure.image.array) 

301 if badPixelMasks is None: 

302 badPixelMasks = ScarletDeblendConfig().badMask 

303 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks) 

304 mask = mExposure.mask.array & badPixels 

305 weights[mask > 0] = 0 

306 

307 if footprint is not None: 

308 # Mask out the pixels outside the footprint 

309 weights *= footprint.spans.asArray() 

310 

311 return lite.LiteObservation( 

312 images=mExposure.image.array, 

313 variance=mExposure.variance.array, 

314 weights=weights, 

315 psfs=psfModels, 

316 model_psf=modelPsf[None, :, :], 

317 convolution_mode=convolutionType, 

318 ) 

319 

320 

321def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None): 

322 """Deblend a parent footprint 

323 

324 Parameters 

325 ---------- 

326 mExposure : `lsst.image.MultibandExposure` 

327 - The multiband exposure containing the image, 

328 mask, and variance data 

329 footprint : `lsst.detection.Footprint` 

330 - The footprint of the parent to deblend 

331 config : `ScarletDeblendConfig` 

332 - Configuration of the deblending task 

333 """ 

334 # Extract coordinates from each MultiColorPeak 

335 bbox = footprint.getBBox() 

336 psfCenter = footprint.getCentroid() 

337 

338 observation = buildLiteObservation( 

339 modelPsf=modelPsf, 

340 psfCenter=psfCenter, 

341 mExposure=mExposure[:, bbox], 

342 footprint=footprint, 

343 badPixelMasks=config.badMask, 

344 useWeights=config.useWeights, 

345 convolutionType=config.convolutionType, 

346 ) 

347 

348 # Convert the centers to pixel coordinates 

349 xmin = bbox.getMinX() 

350 ymin = bbox.getMinY() 

351 centers = [ 

352 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

353 for peak in footprint.peaks 

354 if not isPseudoSource(peak, config.pseudoColumns) 

355 ] 

356 

357 # Initialize the sources 

358 if config.morphImage == "chi2": 

359 sources = lite.init_all_sources_main( 

360 observation, 

361 centers, 

362 min_snr=config.minSNR, 

363 thresh=config.morphThresh, 

364 ) 

365 elif config.morphImage == "wavelet": 

366 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin()) 

367 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)] 

368 sources = lite.init_all_sources_wavelets( 

369 observation, 

370 centers, 

371 use_psf=False, 

372 wavelets=_wavelets, 

373 min_snr=config.minSNR, 

374 ) 

375 else: 

376 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.") 

377 

378 # Set the optimizer 

379 if config.optimizer == "adaprox": 

380 parameterization = partial( 

381 lite.init_adaprox_component, 

382 bg_thresh=config.backgroundThresh, 

383 max_prox_iter=config.maxProxIter, 

384 ) 

385 elif config.optimizer == "fista": 

386 parameterization = partial( 

387 lite.init_fista_component, 

388 bg_thresh=config.backgroundThresh, 

389 ) 

390 else: 

391 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.") 

392 sources = lite.parameterize_sources(sources, observation, parameterization) 

393 

394 # Attach the peak to all of the initialized sources 

395 for k, center in enumerate(centers): 

396 # This is just to make sure that there isn't a coding bug 

397 if len(sources[k].components) > 0 and np.any(sources[k].center != center): 

398 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}") 

399 # Store the record for the peak with the appropriate source 

400 sources[k].detectedPeak = footprint.peaks[k] 

401 

402 blend = lite.LiteBlend(sources, observation) 

403 

404 # Initialize each source with its best fit spectrum 

405 if spectrumInit: 

406 blend.fit_spectra() 

407 

408 # Set the sources that could not be initialized and were skipped 

409 skipped = [src for src in sources if src.is_null] 

410 

411 blend.fit( 

412 max_iter=config.maxIter, 

413 e_rel=config.relativeError, 

414 min_iter=config.minIter, 

415 reweight=False, 

416 ) 

417 

418 # Store the location of the PSF center for storage 

419 blend.psfCenter = (psfCenter.x, psfCenter.y) 

420 

421 return blend, skipped 

422 

423 

424@dataclass 

425class DeblenderMetrics: 

426 """Metrics and measurements made on single sources. 

427 

428 Store deblender metrics to be added as attributes to a scarlet source 

429 before it is converted into a `SourceRecord`. 

430 When DM-34414 is finished this class will be eliminated and the metrics 

431 will be added to the schema using a pipeline task that calculates them 

432 from the stored deconvolved models. 

433 

434 All of the parameters are one dimensional numpy arrays, 

435 with an element for each band in the observed images. 

436 

437 `maxOverlap` is useful as a metric for determining how blended a source 

438 is because if it only overlaps with other sources at or below 

439 the noise level, it is likely to be a mostly isolated source 

440 in the deconvolved model frame. 

441 

442 `fluxOverlapFraction` is potentially more useful than the canonical 

443 "blendedness" (or purity) metric because it accounts for potential 

444 biases created during deblending by not weighting the overlapping 

445 flux with the flux of this sources model. 

446 

447 Attributes 

448 ---------- 

449 maxOverlap : `numpy.ndarray` 

450 The maximum overlap that the source has with its neighbors in 

451 a single pixel. 

452 fluxOverlap : `numpy.ndarray` 

453 The total flux from neighbors overlapping with the current source. 

454 fluxOverlapFraction : `numpy.ndarray` 

455 The fraction of `flux from neighbors/source flux` for a 

456 given source within the source's footprint. 

457 blendedness : `numpy.ndarray` 

458 The metric for determining how blended a source is using the 

459 Bosch et al. 2018 metric for "blendedness." Note that some 

460 surveys use the term "purity," which is `1-blendedness`. 

461 """ 

462 maxOverlap: np.array 

463 fluxOverlap: np.array 

464 fluxOverlapFraction: np.array 

465 blendedness: np.array 

466 

467 

468def setDeblenderMetrics(blend): 

469 """Set metrics that can be used to evalute the deblender accuracy 

470 

471 This function calculates the `DeblenderMetrics` for each source in the 

472 blend, and assigns it to that sources `metrics` property in place. 

473 

474 Parameters 

475 ---------- 

476 blend : `scarlet.lite.Blend` 

477 The blend containing the sources to measure. 

478 """ 

479 # Store the full model of the scene for comparison 

480 blendModel = blend.get_model() 

481 for k, src in enumerate(blend.sources): 

482 # Extract the source model in the full bounding box 

483 model = src.get_model(bbox=blend.bbox) 

484 # The footprint is the 2D array of non-zero pixels in each band 

485 footprint = np.bitwise_or.reduce(model > 0, axis=0) 

486 # Calculate the metrics. 

487 # See `DeblenderMetrics` for a description of each metric. 

488 neighborOverlap = (blendModel-model) * footprint[None, :, :] 

489 maxOverlap = np.max(neighborOverlap, axis=(1, 2)) 

490 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2)) 

491 fluxModel = np.sum(model, axis=(1, 2)) 

492 fluxOverlapFraction = np.zeros((len(model), ), dtype=float) 

493 isFinite = fluxModel > 0 

494 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite] 

495 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2)) 

496 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness) 

497 

498 

499class ScarletDeblendConfig(pexConfig.Config): 

500 """MultibandDeblendConfig 

501 

502 Configuration for the multiband deblender. 

503 The parameters are organized by the parameter types, which are 

504 - Stopping Criteria: Used to determine if the fit has converged 

505 - Position Fitting Criteria: Used to fit the positions of the peaks 

506 - Constraints: Used to apply constraints to the peaks and their components 

507 - Other: Parameters that don't fit into the above categories 

508 """ 

509 # Stopping Criteria 

510 minIter = pexConfig.Field(dtype=int, default=15, 

511 doc="Minimum number of iterations before the optimizer is allowed to stop.") 

512 maxIter = pexConfig.Field(dtype=int, default=300, 

513 doc=("Maximum number of iterations to deblend a single parent")) 

514 relativeError = pexConfig.Field(dtype=float, default=1e-2, 

515 doc=("Change in the loss function between iterations to exit fitter. " 

516 "Typically this is `1e-2` if measurements will be made on the " 

517 "flux re-distributed models and `1e-4` when making measurements " 

518 "on the models themselves.")) 

519 

520 # Constraints 

521 morphThresh = pexConfig.Field(dtype=float, default=1, 

522 doc="Fraction of background RMS a pixel must have" 

523 "to be included in the initial morphology") 

524 # Lite Parameters 

525 # All of these parameters (except version) are only valid if version='lite' 

526 version = pexConfig.ChoiceField( 

527 dtype=str, 

528 default="lite", 

529 allowed={ 

530 "scarlet": "main scarlet version (likely to be deprecated soon)", 

531 "lite": "Optimized version of scarlet for survey data from a single instrument", 

532 }, 

533 doc="The version of scarlet to use.", 

534 ) 

535 optimizer = pexConfig.ChoiceField( 

536 dtype=str, 

537 default="adaprox", 

538 allowed={ 

539 "adaprox": "Proximal ADAM optimization", 

540 "fista": "Accelerated proximal gradient method", 

541 }, 

542 doc="The optimizer to use for fitting parameters and is only used when version='lite'", 

543 ) 

544 morphImage = pexConfig.ChoiceField( 

545 dtype=str, 

546 default="chi2", 

547 allowed={ 

548 "chi2": "Initialize sources on a chi^2 image made from all available bands", 

549 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image", 

550 }, 

551 doc="The type of image to use for initializing the morphology. " 

552 "Must be either 'chi2' or 'wavelet'. " 

553 ) 

554 backgroundThresh = pexConfig.Field( 

555 dtype=float, 

556 default=0.25, 

557 doc="Fraction of background to use for a sparsity threshold. " 

558 "This prevents sources from growing unrealistically outside " 

559 "the parent footprint while still modeling flux correctly " 

560 "for bright sources." 

561 ) 

562 maxProxIter = pexConfig.Field( 

563 dtype=int, 

564 default=1, 

565 doc="Maximum number of proximal operator iterations inside of each " 

566 "iteration of the optimizer. " 

567 "This config field is only used if version='lite' and optimizer='adaprox'." 

568 ) 

569 waveletScales = pexConfig.Field( 

570 dtype=int, 

571 default=5, 

572 doc="Number of wavelet scales to use for wavelet initialization. " 

573 "This field is only used when `version`='lite' and `morphImage`='wavelet'." 

574 ) 

575 

576 # Other scarlet paremeters 

577 useWeights = pexConfig.Field( 

578 dtype=bool, default=True, 

579 doc=("Whether or not use use inverse variance weighting." 

580 "If `useWeights` is `False` then flat weights are used")) 

581 modelPsfSize = pexConfig.Field( 

582 dtype=int, default=11, 

583 doc="Model PSF side length in pixels") 

584 modelPsfSigma = pexConfig.Field( 

585 dtype=float, default=0.8, 

586 doc="Define sigma for the model frame PSF") 

587 minSNR = pexConfig.Field( 

588 dtype=float, default=50, 

589 doc="Minimum Signal to noise to accept the source." 

590 "Sources with lower flux will be initialized with the PSF but updated " 

591 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).") 

592 saveTemplates = pexConfig.Field( 

593 dtype=bool, default=True, 

594 doc="Whether or not to save the SEDs and templates") 

595 processSingles = pexConfig.Field( 

596 dtype=bool, default=True, 

597 doc="Whether or not to process isolated sources in the deblender") 

598 convolutionType = pexConfig.Field( 

599 dtype=str, default="fft", 

600 doc="Type of convolution to render the model to the observations.\n" 

601 "- 'fft': perform convolutions in Fourier space\n" 

602 "- 'real': peform convolutions in real space.") 

603 sourceModel = pexConfig.Field( 

604 dtype=str, default="double", 

605 doc=("How to determine which model to use for sources, from\n" 

606 "- 'single': use a single component for all sources\n" 

607 "- 'double': use a bulge disk model for all sources\n" 

608 "- 'compact': use a single component model, initialzed with a point source morphology, " 

609 " for all sources\n" 

610 "- 'point': use a point-source model for all sources\n" 

611 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"), 

612 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.", 

613 ) 

614 setSpectra = pexConfig.Field( 

615 dtype=bool, default=True, 

616 doc="Whether or not to solve for the best-fit spectra during initialization. " 

617 "This makes initialization slightly longer, as it requires a convolution " 

618 "to set the optimal spectra, but results in a much better initial log-likelihood " 

619 "and reduced total runtime, with convergence in fewer iterations." 

620 "This option is only used when " 

621 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.") 

622 

623 # Mask-plane restrictions 

624 badMask = pexConfig.ListField( 

625 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"], 

626 doc="Whether or not to process isolated sources in the deblender") 

627 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

628 doc="Mask planes to ignore when performing statistics") 

629 maskLimits = pexConfig.DictField( 

630 keytype=str, 

631 itemtype=float, 

632 default={}, 

633 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

634 "Sources violating this limit will not be deblended. " 

635 "If the fraction is `0` then the limit is a single pixel."), 

636 ) 

637 

638 # Size restrictions 

639 maxNumberOfPeaks = pexConfig.Field( 

640 dtype=int, default=200, 

641 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

642 " (<= 0: unlimited)")) 

643 maxFootprintArea = pexConfig.Field( 

644 dtype=int, default=100_000, 

645 doc=("Maximum area for footprints before they are ignored as large; " 

646 "non-positive means no threshold applied")) 

647 maxAreaTimesPeaks = pexConfig.Field( 

648 dtype=int, default=10_000_000, 

649 doc=("Maximum rectangular footprint area * nPeaks in the footprint. " 

650 "This was introduced in DM-33690 to prevent fields that are crowded or have a " 

651 "LSB galaxy that causes memory intensive initialization in scarlet from dominating " 

652 "the overall runtime and/or causing the task to run out of memory. " 

653 "(<= 0: unlimited)") 

654 ) 

655 maxFootprintSize = pexConfig.Field( 

656 dtype=int, default=0, 

657 doc=("Maximum linear dimension for footprints before they are ignored " 

658 "as large; non-positive means no threshold applied")) 

659 minFootprintAxisRatio = pexConfig.Field( 

660 dtype=float, default=0.0, 

661 doc=("Minimum axis ratio for footprints before they are ignored " 

662 "as large; non-positive means no threshold applied")) 

663 maxSpectrumCutoff = pexConfig.Field( 

664 dtype=int, default=1_000_000, 

665 doc=("Maximum number of pixels * number of sources in a blend. " 

666 "This is different than `maxFootprintArea` because this isn't " 

667 "the footprint area but the area of the bounding box that " 

668 "contains the footprint, and is also multiplied by the number of" 

669 "sources in the footprint. This prevents large skinny blends with " 

670 "a high density of sources from running out of memory. " 

671 "If `maxSpectrumCutoff == -1` then there is no cutoff.") 

672 ) 

673 # Failure modes 

674 fallback = pexConfig.Field( 

675 dtype=bool, default=True, 

676 doc="Whether or not to fallback to a smaller number of components if a source does not initialize" 

677 ) 

678 notDeblendedMask = pexConfig.Field( 

679 dtype=str, default="NOT_DEBLENDED", optional=True, 

680 doc="Mask name for footprints not deblended, or None") 

681 catchFailures = pexConfig.Field( 

682 dtype=bool, default=True, 

683 doc=("If True, catch exceptions thrown by the deblender, log them, " 

684 "and set a flag on the parent, instead of letting them propagate up")) 

685 

686 # Other options 

687 columnInheritance = pexConfig.DictField( 

688 keytype=str, itemtype=str, default={ 

689 "deblend_nChild": "deblend_parentNChild", 

690 "deblend_nPeaks": "deblend_parentNPeaks", 

691 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag", 

692 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag", 

693 }, 

694 doc="Columns to pass from the parent to the child. " 

695 "The key is the name of the column for the parent record, " 

696 "the value is the name of the column to use for the child." 

697 ) 

698 pseudoColumns = pexConfig.ListField( 

699 dtype=str, default=['merge_peak_sky', 'sky_source'], 

700 doc="Names of flags which should never be deblended." 

701 ) 

702 

703 # Testing options 

704 # Some obs packages and ci packages run the full pipeline on a small 

705 # subset of data to test that the pipeline is functioning properly. 

706 # This is not meant as scientific validation, so it can be useful 

707 # to only run on a small subset of the data that is large enough to 

708 # test the desired pipeline features but not so long that the deblender 

709 # is the tall pole in terms of execution times. 

710 useCiLimits = pexConfig.Field( 

711 dtype=bool, default=False, 

712 doc="Limit the number of sources deblended for CI to prevent long build times") 

713 ciDeblendChildRange = pexConfig.ListField( 

714 dtype=int, default=[5, 10], 

715 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated." 

716 "If `useCiLimits==False` then this parameter is ignored.") 

717 ciNumParentsToDeblend = pexConfig.Field( 

718 dtype=int, default=10, 

719 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count " 

720 "within `ciDebledChildRange`. " 

721 "If `useCiLimits==False` then this parameter is ignored.") 

722 

723 

724class ScarletDeblendTask(pipeBase.Task): 

725 """ScarletDeblendTask 

726 

727 Split blended sources into individual sources. 

728 

729 This task has no return value; it only modifies the SourceCatalog in-place. 

730 """ 

731 ConfigClass = ScarletDeblendConfig 

732 _DefaultName = "scarletDeblend" 

733 

734 def __init__(self, schema, peakSchema=None, **kwargs): 

735 """Create the task, adding necessary fields to the given schema. 

736 

737 Parameters 

738 ---------- 

739 schema : `lsst.afw.table.schema.schema.Schema` 

740 Schema object for measurement fields; will be modified in-place. 

741 peakSchema : `lsst.afw.table.schema.schema.Schema` 

742 Schema of Footprint Peaks that will be passed to the deblender. 

743 Any fields beyond the PeakTable minimal schema will be transferred 

744 to the main source Schema. If None, no fields will be transferred 

745 from the Peaks. 

746 filters : list of str 

747 Names of the filters used for the eposures. This is needed to store 

748 the SED as a field 

749 **kwargs 

750 Passed to Task.__init__. 

751 """ 

752 pipeBase.Task.__init__(self, **kwargs) 

753 

754 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

755 if peakSchema is None: 

756 # In this case, the peakSchemaMapper will transfer nothing, but 

757 # we'll still have one 

758 # to simplify downstream code 

759 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

760 else: 

761 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

762 for item in peakSchema: 

763 if item.key not in peakMinimalSchema: 

764 self.peakSchemaMapper.addMapping(item.key, item.field) 

765 # Because SchemaMapper makes a copy of the output schema 

766 # you give its ctor, it isn't updating this Schema in 

767 # place. That's probably a design flaw, but in the 

768 # meantime, we'll keep that schema in sync with the 

769 # peakSchemaMapper.getOutputSchema() manually, by adding 

770 # the same fields to both. 

771 schema.addField(item.field) 

772 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

773 self._addSchemaKeys(schema) 

774 self.schema = schema 

775 self.toCopyFromParent = [item.key for item in self.schema 

776 if item.field.getName().startswith("merge_footprint")] 

777 

778 def _addSchemaKeys(self, schema): 

779 """Add deblender specific keys to the schema 

780 """ 

781 # Parent (blend) fields 

782 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms') 

783 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge') 

784 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

785 doc='Number of children this object has (defaults to 0)') 

786 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32, 

787 doc="Number of initial peaks in the blend. " 

788 "This includes peaks that may have been culled " 

789 "during deblending or failed to deblend") 

790 # Skipped flags 

791 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

792 doc="Deblender skipped this source") 

793 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag', 

794 doc='The source has only a single peak ' 

795 'and was not deblended') 

796 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag', 

797 doc='The source is identified as a "pseudo" source and ' 

798 'was not deblended') 

799 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

800 doc='Source had too many peaks; ' 

801 'only the brightest were included') 

802 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

803 doc='Parent footprint covered too many pixels') 

804 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

805 doc='Parent footprint had too many masked pixels') 

806 # Convergence flags 

807 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag', 

808 doc='scarlet sed optimization did not converge before' 

809 'config.maxIter') 

810 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag', 

811 doc='scarlet morph optimization did not converge before' 

812 'config.maxIter') 

813 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag', 

814 type='Flag', 

815 doc='at least one source in the blend' 

816 'failed to converge') 

817 # Error flags 

818 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

819 doc="Deblending failed on source") 

820 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25, 

821 doc='Name of error if the blend failed') 

822 # Deblended source fields 

823 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center", 

824 doc="Center used to apply constraints in scarlet", 

825 unit="pixel") 

826 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32, 

827 doc="ID of the peak in the parent footprint. " 

828 "This is not unique, but the combination of 'parent'" 

829 "and 'peakId' should be for all child sources. " 

830 "Top level blends with no parents have 'peakId=0'") 

831 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count', 

832 doc="The instFlux at the peak position of deblended mode") 

833 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25, 

834 doc="The type of model used, for example " 

835 "MultiExtendedSource, SingleExtendedSource, PointSource") 

836 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32, 

837 doc="deblend_nPeaks from this records parent.") 

838 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32, 

839 doc="deblend_nChild from this records parent.") 

840 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32, 

841 doc="Flux measurement from scarlet") 

842 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32, 

843 doc="Final logL, used to identify regressions in scarlet.") 

844 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag', 

845 doc='Source had flux on the edge of the parent footprint') 

846 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag', 

847 doc="True when scarlet initializes sources " 

848 "in the blend with a more accurate spectrum. " 

849 "The algorithm uses a lot of memory, " 

850 "so large dense blends will use " 

851 "a less accurate initialization.") 

852 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32, 

853 doc="Number of components in a ScarletLiteSource. " 

854 "If `config.version != 'lite'`then " 

855 "this column is set to zero.") 

856 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

857 doc='Deblender thought this source looked like a PSF') 

858 # Blendedness/classification metrics 

859 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32, 

860 doc="Maximum overlap with all of the other neighbors flux " 

861 "combined." 

862 "This is useful as a metric for determining how blended a " 

863 "source is because if it only overlaps with other sources " 

864 "at or below the noise level, it is likely to be a mostly " 

865 "isolated source in the deconvolved model frame.") 

866 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32, 

867 doc="This is the total flux from neighboring objects that " 

868 "overlaps with this source.") 

869 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32, 

870 doc="This is the fraction of " 

871 "`flux from neighbors/source flux` " 

872 "for a given source within the source's" 

873 "footprint.") 

874 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32, 

875 doc="The Bosch et al. 2018 metric for 'blendedness.' ") 

876 

877 @timeMethod 

878 def run(self, mExposure, mergedSources): 

879 """Get the psf from each exposure and then run deblend(). 

880 

881 Parameters 

882 ---------- 

883 mExposure : `MultibandExposure` 

884 The exposures should be co-added images of the same 

885 shape and region of the sky. 

886 mergedSources : `SourceCatalog` 

887 The merged `SourceCatalog` that contains parent footprints 

888 to (potentially) deblend. 

889 

890 Returns 

891 ------- 

892 templateCatalogs: dict 

893 Keys are the names of the filters and the values are 

894 `lsst.afw.table.source.source.SourceCatalog`'s. 

895 These are catalogs with heavy footprints that are the templates 

896 created by the multiband templates. 

897 """ 

898 return self.deblend(mExposure, mergedSources) 

899 

900 @timeMethod 

901 def deblend(self, mExposure, catalog): 

902 """Deblend a data cube of multiband images 

903 

904 Parameters 

905 ---------- 

906 mExposure : `MultibandExposure` 

907 The exposures should be co-added images of the same 

908 shape and region of the sky. 

909 catalog : `SourceCatalog` 

910 The merged `SourceCatalog` that contains parent footprints 

911 to (potentially) deblend. The new deblended sources are 

912 appended to this catalog in place. 

913 

914 Returns 

915 ------- 

916 catalogs : `dict` or `None` 

917 Keys are the names of the filters and the values are 

918 `lsst.afw.table.source.source.SourceCatalog`'s. 

919 These are catalogs with heavy footprints that are the templates 

920 created by the multiband templates. 

921 """ 

922 import time 

923 

924 # Cull footprints if required by ci 

925 if self.config.useCiLimits: 

926 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.", 

927 len(catalog)) 

928 # Select parents with a number of children in the range 

929 # config.ciDeblendChildRange 

930 minChildren, maxChildren = self.config.ciDeblendChildRange 

931 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog]) 

932 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0] 

933 if len(childrenInRange) < self.config.ciNumParentsToDeblend: 

934 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range " 

935 "indicated by ciDeblendChildRange. Adjust this range to include more " 

936 "parents.") 

937 # Keep all of the isolated parents and the first 

938 # `ciNumParentsToDeblend` children 

939 parents = nPeaks == 1 

940 children = np.zeros((len(catalog),), dtype=bool) 

941 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True 

942 catalog = catalog[parents | children] 

943 # We need to update the IdFactory, otherwise the the source ids 

944 # will not be sequential 

945 idFactory = catalog.getIdFactory() 

946 maxId = np.max(catalog["id"]) 

947 idFactory.notify(maxId) 

948 

949 filters = mExposure.filters 

950 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure)) 

951 periodicLog = PeriodicLogger(self.log) 

952 

953 # Create a set of wavelet coefficients if using wavelet initialization 

954 if self.config.version == "lite" and self.config.morphImage == "wavelet": 

955 images = mExposure.image.array 

956 variance = mExposure.variance.array 

957 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales) 

958 else: 

959 wavelets = None 

960 

961 # Add the NOT_DEBLENDED mask to the mask plane in each band 

962 if self.config.notDeblendedMask: 

963 for mask in mExposure.mask: 

964 mask.addMaskPlane(self.config.notDeblendedMask) 

965 

966 # Initialize the persistable data model 

967 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma) 

968 dataModel = ScarletModelData(filters, modelPsf) 

969 

970 nParents = len(catalog) 

971 nDeblendedParents = 0 

972 skippedParents = [] 

973 for parentIndex in range(nParents): 

974 parent = catalog[parentIndex] 

975 foot = parent.getFootprint() 

976 bbox = foot.getBBox() 

977 peaks = foot.getPeaks() 

978 

979 # Since we use the first peak for the parent object, we should 

980 # propagate its flags to the parent source. 

981 parent.assign(peaks[0], self.peakSchemaMapper) 

982 

983 # Block of conditions for skipping a parent with multiple children 

984 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None: 

985 self._skipParent(parent, *skipArgs) 

986 skippedParents.append(parentIndex) 

987 continue 

988 

989 nDeblendedParents += 1 

990 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks)) 

991 # Run the deblender 

992 blendError = None 

993 

994 # Choose whether or not to use improved spectral initialization. 

995 # This significantly cuts down on the number of iterations 

996 # that the optimizer needs and usually results in a better 

997 # fit. 

998 # But using least squares on a very large blend causes memory 

999 # issues, so it is not done for large blends 

1000 if self.config.setSpectra: 

1001 if self.config.maxSpectrumCutoff <= 0: 

1002 spectrumInit = True 

1003 else: 

1004 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff 

1005 else: 

1006 spectrumInit = False 

1007 

1008 try: 

1009 t0 = time.monotonic() 

1010 # Build the parameter lists with the same ordering 

1011 if self.config.version == "scarlet": 

1012 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit) 

1013 elif self.config.version == "lite": 

1014 blend, skipped = deblend_lite( 

1015 mExposure=mExposure, 

1016 modelPsf=modelPsf, 

1017 footprint=foot, 

1018 config=self.config, 

1019 spectrumInit=spectrumInit, 

1020 wavelets=wavelets, 

1021 ) 

1022 tf = time.monotonic() 

1023 runtime = (tf-t0)*1000 

1024 converged = _checkBlendConvergence(blend, self.config.relativeError) 

1025 # Store the number of components in the blend 

1026 if self.config.version == "lite": 

1027 nComponents = len(blend.components) 

1028 else: 

1029 nComponents = 0 

1030 nChild = len(blend.sources) 

1031 # Catch all errors and filter out the ones that we know about 

1032 except Exception as e: 

1033 blendError = type(e).__name__ 

1034 if isinstance(e, ScarletGradientError): 

1035 parent.set(self.iterKey, e.iterations) 

1036 elif not isinstance(e, IncompleteDataError): 

1037 blendError = "UnknownError" 

1038 if self.config.catchFailures: 

1039 # Make it easy to find UnknownErrors in the log file 

1040 self.log.warn("UnknownError") 

1041 import traceback 

1042 traceback.print_exc() 

1043 else: 

1044 raise 

1045 

1046 self._skipParent( 

1047 parent=parent, 

1048 skipKey=self.deblendFailedKey, 

1049 logMessage=f"Unable to deblend source {parent.getId}: {blendError}", 

1050 ) 

1051 parent.set(self.deblendErrorKey, blendError) 

1052 skippedParents.append(parentIndex) 

1053 continue 

1054 

1055 # Update the parent record with the deblending results 

1056 if self.config.version == "scarlet": 

1057 logL = -blend.loss[-1] + blend.observations[0].log_norm 

1058 elif self.config.version == "lite": 

1059 logL = blend.loss[-1] 

1060 self._updateParentRecord( 

1061 parent=parent, 

1062 nPeaks=len(peaks), 

1063 nChild=nChild, 

1064 nComponents=nComponents, 

1065 runtime=runtime, 

1066 iterations=len(blend.loss), 

1067 logL=logL, 

1068 spectrumInit=spectrumInit, 

1069 converged=converged, 

1070 ) 

1071 

1072 # Add each deblended source to the catalog 

1073 for k, scarletSource in enumerate(blend.sources): 

1074 # Skip any sources with no flux or that scarlet skipped because 

1075 # it could not initialize 

1076 if k in skipped or (self.config.version == "lite" and scarletSource.is_null): 

1077 # No need to propagate anything 

1078 continue 

1079 parent.set(self.deblendSkippedKey, False) 

1080 

1081 # Add all fields except the HeavyFootprint to the 

1082 # source record 

1083 sourceRecord = self._addChild( 

1084 parent=parent, 

1085 peak=scarletSource.detectedPeak, 

1086 catalog=catalog, 

1087 scarletSource=scarletSource, 

1088 ) 

1089 scarletSource.recordId = sourceRecord.getId() 

1090 scarletSource.peakId = scarletSource.detectedPeak.getId() 

1091 

1092 # Store the blend information so that it can be persisted 

1093 if self.config.version == "lite": 

1094 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin()) 

1095 else: 

1096 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin()) 

1097 dataModel.blends[parent.getId()] = blendData 

1098 

1099 # Log a message if it has been a while since the last log. 

1100 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents) 

1101 

1102 # Clear the cached values in scarlet to clear out memory 

1103 scarlet.cache.Cache._cache = {} 

1104 

1105 # Update the mExposure mask with the footprint of skipped parents 

1106 if self.config.notDeblendedMask: 

1107 for mask in mExposure.mask: 

1108 for parentIndex in skippedParents: 

1109 fp = catalog[parentIndex].getFootprint() 

1110 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask)) 

1111 

1112 self.log.info("Deblender results: of %d parent sources, %d were deblended, " 

1113 "creating %d children, for a total of %d sources", 

1114 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog)) 

1115 return catalog, dataModel 

1116 

1117 def _isLargeFootprint(self, footprint): 

1118 """Returns whether a Footprint is large 

1119 

1120 'Large' is defined by thresholds on the area, size and axis ratio, 

1121 and total area of the bounding box multiplied by 

1122 the number of children. 

1123 These may be disabled independently by configuring them to be 

1124 non-positive. 

1125 """ 

1126 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

1127 return True 

1128 if self.config.maxFootprintSize > 0: 

1129 bbox = footprint.getBBox() 

1130 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

1131 return True 

1132 if self.config.minFootprintAxisRatio > 0: 

1133 axes = afwEll.Axes(footprint.getShape()) 

1134 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

1135 return True 

1136 if self.config.maxAreaTimesPeaks > 0: 

1137 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks: 

1138 return True 

1139 return False 

1140 

1141 def _isMasked(self, footprint, mExposure): 

1142 """Returns whether the footprint violates the mask limits 

1143 

1144 Parameters 

1145 ---------- 

1146 footprint : `lsst.afw.detection.Footprint` 

1147 The footprint to check for masked pixels 

1148 mMask : `lsst.afw.image.MaskX` 

1149 The mask plane to check for masked pixels in the `footprint`. 

1150 

1151 Returns 

1152 ------- 

1153 isMasked : `bool` 

1154 `True` if `self.config.maskPlaneLimits` is less than the 

1155 fraction of pixels for a given mask in 

1156 `self.config.maskLimits`. 

1157 """ 

1158 bbox = footprint.getBBox() 

1159 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0) 

1160 size = float(footprint.getArea()) 

1161 for maskName, limit in self.config.maskLimits.items(): 

1162 maskVal = mExposure.mask.getPlaneBitMask(maskName) 

1163 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin()) 

1164 # spanset of masked pixels 

1165 maskedSpan = footprint.spans.intersect(_mask, maskVal) 

1166 if (maskedSpan.getArea())/size > limit: 

1167 return True 

1168 return False 

1169 

1170 def _skipParent(self, parent, skipKey, logMessage): 

1171 """Update a parent record that is not being deblended. 

1172 

1173 This is a fairly trivial function but is implemented to ensure 

1174 that a skipped parent updates the appropriate columns 

1175 consistently, and always has a flag to mark the reason that 

1176 it is being skipped. 

1177 

1178 Parameters 

1179 ---------- 

1180 parent : `lsst.afw.table.source.source.SourceRecord` 

1181 The parent record to flag as skipped. 

1182 skipKey : `bool` 

1183 The name of the flag to mark the reason for skipping. 

1184 logMessage : `str` 

1185 The message to display in a log.trace when a source 

1186 is skipped. 

1187 """ 

1188 if logMessage is not None: 

1189 self.log.trace(logMessage) 

1190 self._updateParentRecord( 

1191 parent=parent, 

1192 nPeaks=len(parent.getFootprint().peaks), 

1193 nChild=0, 

1194 nComponents=0, 

1195 runtime=np.nan, 

1196 iterations=0, 

1197 logL=np.nan, 

1198 spectrumInit=False, 

1199 converged=False, 

1200 ) 

1201 

1202 # Mark the source as skipped by the deblender and 

1203 # flag the reason why. 

1204 parent.set(self.deblendSkippedKey, True) 

1205 parent.set(skipKey, True) 

1206 

1207 def _checkSkipped(self, parent, mExposure): 

1208 """Update a parent record that is not being deblended. 

1209 

1210 This is a fairly trivial function but is implemented to ensure 

1211 that a skipped parent updates the appropriate columns 

1212 consistently, and always has a flag to mark the reason that 

1213 it is being skipped. 

1214 

1215 Parameters 

1216 ---------- 

1217 parent : `lsst.afw.table.source.source.SourceRecord` 

1218 The parent record to flag as skipped. 

1219 mExposure : `MultibandExposure` 

1220 The exposures should be co-added images of the same 

1221 shape and region of the sky. 

1222 Returns 

1223 ------- 

1224 skip: `bool` 

1225 `True` if the deblender will skip the parent 

1226 """ 

1227 skipKey = None 

1228 skipMessage = None 

1229 footprint = parent.getFootprint() 

1230 if len(footprint.peaks) < 2 and not self.config.processSingles: 

1231 # Skip isolated sources unless processSingles is turned on. 

1232 # Note: this does not flag isolated sources as skipped or 

1233 # set the NOT_DEBLENDED mask in the exposure, 

1234 # since these aren't really any skipped blends. 

1235 skipKey = self.isolatedParentKey 

1236 elif isPseudoSource(parent, self.config.pseudoColumns): 

1237 # We also skip pseudo sources, like sky objects, which 

1238 # are intended to be skipped. 

1239 skipKey = self.pseudoKey 

1240 if self._isLargeFootprint(footprint): 

1241 # The footprint is above the maximum footprint size limit 

1242 skipKey = self.tooBigKey 

1243 skipMessage = f"Parent {parent.getId()}: skipping large footprint" 

1244 elif self._isMasked(footprint, mExposure): 

1245 # The footprint exceeds the maximum number of masked pixels 

1246 skipKey = self.maskedKey 

1247 skipMessage = f"Parent {parent.getId()}: skipping masked footprint" 

1248 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks: 

1249 # Unlike meas_deblender, in scarlet we skip the entire blend 

1250 # if the number of peaks exceeds max peaks, since neglecting 

1251 # to model any peaks often results in catastrophic failure 

1252 # of scarlet to generate models for the brighter sources. 

1253 skipKey = self.tooManyPeaksKey 

1254 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks" 

1255 if skipKey is not None: 

1256 return (skipKey, skipMessage) 

1257 return None 

1258 

1259 def setSkipFlags(self, mExposure, catalog): 

1260 """Set the skip flags for all of the parent sources 

1261 

1262 This is mostly used for testing which parent sources will be deblended 

1263 and which will be skipped based on the current configuration options. 

1264 Skipped sources will have the appropriate flags set in place in the 

1265 catalog. 

1266 

1267 Parameters 

1268 ---------- 

1269 mExposure : `MultibandExposure` 

1270 The exposures should be co-added images of the same 

1271 shape and region of the sky. 

1272 catalog : `SourceCatalog` 

1273 The merged `SourceCatalog` that contains parent footprints 

1274 to (potentially) deblend. The new deblended sources are 

1275 appended to this catalog in place. 

1276 """ 

1277 for src in catalog: 

1278 if skipArgs := self._checkSkipped(src, mExposure) is not None: 

1279 self._skipParent(src, *skipArgs) 

1280 

1281 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents, 

1282 runtime, iterations, logL, spectrumInit, converged): 

1283 """Update a parent record in all of the single band catalogs. 

1284 

1285 Ensure that all locations that update a parent record, 

1286 whether it is skipped or updated after deblending, 

1287 update all of the appropriate columns. 

1288 

1289 Parameters 

1290 ---------- 

1291 parent : `lsst.afw.table.source.source.SourceRecord` 

1292 The parent record to update. 

1293 nPeaks : `int` 

1294 Number of peaks in the parent footprint. 

1295 nChild : `int` 

1296 Number of children deblended from the parent. 

1297 This may differ from `nPeaks` if some of the peaks 

1298 were culled and have no deblended model. 

1299 nComponents : `int` 

1300 Total number of components in the parent. 

1301 This is usually different than the number of children, 

1302 since it is common for a single source to have multiple 

1303 components. 

1304 runtime : `float` 

1305 Total runtime for deblending. 

1306 iterations : `int` 

1307 Total number of iterations in scarlet before convergence. 

1308 logL : `float` 

1309 Final log likelihood of the blend. 

1310 spectrumInit : `bool` 

1311 True when scarlet used `set_spectra` to initialize all 

1312 sources with better initial intensities. 

1313 converged : `bool` 

1314 True when the optimizer reached convergence before 

1315 reaching the maximum number of iterations. 

1316 """ 

1317 parent.set(self.nPeaksKey, nPeaks) 

1318 parent.set(self.nChildKey, nChild) 

1319 parent.set(self.nComponentsKey, nComponents) 

1320 parent.set(self.runtimeKey, runtime) 

1321 parent.set(self.iterKey, iterations) 

1322 parent.set(self.scarletLogLKey, logL) 

1323 parent.set(self.scarletSpectrumInitKey, spectrumInit) 

1324 parent.set(self.blendConvergenceFailedFlagKey, converged) 

1325 

1326 def _addChild(self, parent, peak, catalog, scarletSource): 

1327 """Add a child to a catalog. 

1328 

1329 This creates a new child in the source catalog, 

1330 assigning it a parent id, and adding all columns 

1331 that are independent across all filter bands. 

1332 

1333 Parameters 

1334 ---------- 

1335 parent : `lsst.afw.table.source.source.SourceRecord` 

1336 The parent of the new child record. 

1337 peak : `lsst.afw.table.PeakRecord` 

1338 The peak record for the peak from the parent peak catalog. 

1339 catalog : `lsst.afw.table.source.source.SourceCatalog` 

1340 The merged `SourceCatalog` that contains parent footprints 

1341 to (potentially) deblend. 

1342 scarletSource : `scarlet.Component` 

1343 The scarlet model for the new source record. 

1344 """ 

1345 src = catalog.addNew() 

1346 for key in self.toCopyFromParent: 

1347 src.set(key, parent.get(key)) 

1348 # The peak catalog is the same for all bands, 

1349 # so we just use the first peak catalog 

1350 src.assign(peak, self.peakSchemaMapper) 

1351 src.setParent(parent.getId()) 

1352 src.set(self.nPeaksKey, 1) 

1353 # Set the psf key based on whether or not the source was 

1354 # deblended using the PointSource model. 

1355 # This key is not that useful anymore since we now keep track of 

1356 # `modelType`, but we continue to propagate it in case code downstream 

1357 # is expecting it. 

1358 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource") 

1359 src.set(self.modelTypeKey, scarletSource.__class__.__name__) 

1360 # We set the runtime to zero so that summing up the 

1361 # runtime column will give the total time spent 

1362 # running the deblender for the catalog. 

1363 src.set(self.runtimeKey, 0) 

1364 

1365 # Set the position of the peak from the parent footprint 

1366 # This will make it easier to match the same source across 

1367 # deblenders and across observations, where the peak 

1368 # position is unlikely to change unless enough time passes 

1369 # for a source to move on the sky. 

1370 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"])) 

1371 src.set(self.peakIdKey, peak["id"]) 

1372 

1373 # Store the number of components for the source 

1374 src.set(self.nComponentsKey, len(scarletSource.components)) 

1375 

1376 # Propagate columns from the parent to the child 

1377 for parentColumn, childColumn in self.config.columnInheritance.items(): 

1378 src.set(childColumn, parent.get(parentColumn)) 

1379 

1380 return src