Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%

448 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-05 19:34 +0000

1# This file is part of meas_extensions_scarlet. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from dataclasses import dataclass 

23from functools import partial 

24import logging 

25import numpy as np 

26import scarlet 

27from scarlet.psf import ImagePSF, GaussianPSF 

28from scarlet import Blend, Frame, Observation 

29from scarlet.renderer import ConvolutionRenderer 

30from scarlet.detect import get_detect_wavelets 

31from scarlet.initialization import init_all_sources 

32from scarlet import lite 

33 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36from lsst.geom import Point2I, Point2D 

37import lsst.afw.geom.ellipses as afwEll 

38import lsst.afw.image as afwImage 

39import lsst.afw.detection as afwDet 

40import lsst.afw.table as afwTable 

41from lsst.utils.logging import PeriodicLogger 

42from lsst.utils.timer import timeMethod 

43 

44from .source import bboxToScarletBox 

45from .io import ScarletModelData, scarletToData, scarletLiteToData 

46 

47# Scarlet and proxmin have a different definition of log levels than the stack, 

48# so even "warnings" occur far more often than we would like. 

49# So for now we only display scarlet and proxmin errors, as all other 

50# scarlet outputs would be considered "TRACE" by our standards. 

51scarletLogger = logging.getLogger("scarlet") 

52scarletLogger.setLevel(logging.ERROR) 

53proxminLogger = logging.getLogger("proxmin") 

54proxminLogger.setLevel(logging.ERROR) 

55 

56__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"] 

57 

58logger = logging.getLogger(__name__) 

59 

60 

61class IncompleteDataError(Exception): 

62 """The PSF could not be computed due to incomplete data 

63 """ 

64 pass 

65 

66 

67class ScarletGradientError(Exception): 

68 """An error occurred during optimization 

69 

70 This error occurs when the optimizer encounters 

71 a NaN value while calculating the gradient. 

72 """ 

73 def __init__(self, iterations, sources): 

74 self.iterations = iterations 

75 self.sources = sources 

76 msg = ("ScalarGradientError in iteration {0}. " 

77 "NaN values introduced in sources {1}") 

78 self.message = msg.format(iterations, sources) 

79 

80 def __str__(self): 

81 return self.message 

82 

83 

84def _checkBlendConvergence(blend, f_rel): 

85 """Check whether or not a blend has converged 

86 """ 

87 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1]) 

88 convergence = f_rel * np.abs(blend.loss[-1]) 

89 return deltaLoss < convergence 

90 

91 

92def isPseudoSource(source, pseudoColumns): 

93 """Check if a source is a pseudo source. 

94 

95 This is mostly for skipping sky objects, 

96 but any other column can also be added to disable 

97 deblending on a parent or individual source when 

98 set to `True`. 

99 

100 Parameters 

101 ---------- 

102 source : `lsst.afw.table.source.source.SourceRecord` 

103 The source to check for the pseudo bit. 

104 pseudoColumns : `list` of `str` 

105 A list of columns to check for pseudo sources. 

106 """ 

107 isPseudo = False 

108 for col in pseudoColumns: 

109 try: 

110 isPseudo |= source[col] 

111 except KeyError: 

112 pass 

113 return isPseudo 

114 

115 

116def deblend(mExposure, footprint, config, spectrumInit): 

117 """Deblend a parent footprint 

118 

119 Parameters 

120 ---------- 

121 mExposure : `lsst.image.MultibandExposure` 

122 The multiband exposure containing the image, 

123 mask, and variance data. 

124 footprint : `lsst.detection.Footprint` 

125 The footprint of the parent to deblend. 

126 config : `ScarletDeblendConfig` 

127 Configuration of the deblending task. 

128 spectrumInit : `bool` 

129 Whether or not to initialize the spectrum. 

130 

131 Returns 

132 ------- 

133 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData` 

134 The persistable representation of a `scarlet.Blend`. 

135 skipped : `list` of `int` 

136 The indices of any children that failed to initialize 

137 and were skipped. 

138 spectrumInit : `bool` 

139 Whether or not all of the sources were initialized by jointly 

140 fitting their SED's. This provides a better initialization 

141 but created memory issues when a blend is too large or 

142 contains too many sources. 

143 """ 

144 # Extract coordinates from each MultiColorPeak 

145 bbox = footprint.getBBox() 

146 

147 # Create the data array from the masked images 

148 images = mExposure.image[:, bbox].array 

149 

150 # Use the inverse variance as the weights 

151 if config.useWeights: 

152 weights = 1/mExposure.variance[:, bbox].array 

153 else: 

154 weights = np.ones_like(images) 

155 badPixels = mExposure.mask.getPlaneBitMask(config.badMask) 

156 mask = mExposure.mask[:, bbox].array & badPixels 

157 weights[mask > 0] = 0 

158 

159 # Mask out the pixels outside the footprint 

160 weights *= footprint.spans.asArray() 

161 

162 psfCenter = footprint.getCentroid() 

163 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32) 

164 psfs = ImagePSF(psfs) 

165 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters)) 

166 

167 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters) 

168 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters) 

169 if config.convolutionType == "fft": 

170 observation.match(frame) 

171 elif config.convolutionType == "real": 

172 renderer = ConvolutionRenderer(observation, frame, convolution_type="real") 

173 observation.match(frame, renderer=renderer) 

174 else: 

175 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType)) 

176 

177 assert(config.sourceModel in ["single", "double", "compact", "fit"]) 

178 

179 # Set the appropriate number of components 

180 if config.sourceModel == "single": 

181 maxComponents = 1 

182 elif config.sourceModel == "double": 

183 maxComponents = 2 

184 elif config.sourceModel == "compact": 

185 maxComponents = 0 

186 elif config.sourceModel == "point": 

187 raise NotImplementedError("Point source photometry is currently not implemented") 

188 elif config.sourceModel == "fit": 

189 # It is likely in the future that there will be some heuristic 

190 # used to determine what type of model to use for each source, 

191 # but that has not yet been implemented (see DM-22551) 

192 raise NotImplementedError("sourceModel 'fit' has not been implemented yet") 

193 

194 # Convert the centers to pixel coordinates 

195 xmin = bbox.getMinX() 

196 ymin = bbox.getMinY() 

197 centers = [ 

198 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

199 for peak in footprint.peaks 

200 if not isPseudoSource(peak, config.pseudoColumns) 

201 ] 

202 

203 # Only deblend sources that can be initialized 

204 sources, skipped = init_all_sources( 

205 frame=frame, 

206 centers=centers, 

207 observations=observation, 

208 thresh=config.morphThresh, 

209 max_components=maxComponents, 

210 min_snr=config.minSNR, 

211 shifting=False, 

212 fallback=config.fallback, 

213 silent=config.catchFailures, 

214 set_spectra=spectrumInit, 

215 ) 

216 

217 # Attach the peak to all of the initialized sources 

218 srcIndex = 0 

219 for k, center in enumerate(centers): 

220 if k not in skipped: 

221 # This is just to make sure that there isn't a coding bug 

222 assert np.all(sources[srcIndex].center == center) 

223 # Store the record for the peak with the appropriate source 

224 sources[srcIndex].detectedPeak = footprint.peaks[k] 

225 srcIndex += 1 

226 

227 # Create the blend and attempt to optimize it 

228 blend = Blend(sources, observation) 

229 try: 

230 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError) 

231 except ArithmeticError: 

232 # This occurs when a gradient update produces a NaN value 

233 # This is usually due to a source initialized with a 

234 # negative SED or no flux, often because the peak 

235 # is a noise fluctuation in one band and not a real source. 

236 iterations = len(blend.loss) 

237 failedSources = [] 

238 for k, src in enumerate(sources): 

239 if np.any(~np.isfinite(src.get_model())): 

240 failedSources.append(k) 

241 raise ScarletGradientError(iterations, failedSources) 

242 

243 # Store the location of the PSF center for storage 

244 blend.psfCenter = (psfCenter.x, psfCenter.y) 

245 

246 return blend, skipped 

247 

248 

249def buildLiteObservation( 

250 modelPsf, 

251 psfCenter, 

252 mExposure, 

253 footprint=None, 

254 badPixelMasks=None, 

255 useWeights=True, 

256 convolutionType="real", 

257): 

258 """Generate a LiteObservation from a set of parameters. 

259 

260 Make the generation and reconstruction of a scarlet model consistent 

261 by building a `LiteObservation` from a set of parameters. 

262 

263 Parameters 

264 ---------- 

265 modelPsf : `numpy.ndarray` 

266 The 2D model of the PSF in the partially deconvolved space. 

267 psfCenter : `tuple` or `Point2I` or `Point2D` 

268 The location `(x, y)` used as the center of the PSF. 

269 mExposure : `lsst.afw.image.multiband.MultibandExposure` 

270 The multi-band exposure that the model represents. 

271 If `mExposure` is `None` then no image, variance, or weights are 

272 attached to the observation. 

273 footprint : `lsst.afw.detection.Footprint` 

274 The footprint that is being fit. 

275 If `Footprint` is `None` then the weights are not updated to mask 

276 out pixels not contained in the footprint. 

277 badPixelMasks : `list` of `str` 

278 The keys from the bit mask plane used to mask out pixels 

279 during the fit. 

280 If `badPixelMasks` is `None` then the default values from 

281 `ScarletDeblendConfig.badMask` is used. 

282 useWeights : `bool` 

283 Whether or not fitting should use inverse variance weights to 

284 calculate the log-likelihood. 

285 convolutionType : `str` 

286 The type of convolution to use (either "real" or "fft"). 

287 When reconstructing an image it is advised to use "real" to avoid 

288 polluting the footprint with 

289 

290 Returns 

291 ------- 

292 observation : `scarlet.lite.LiteObservation` 

293 The observation constructed from the input parameters. 

294 """ 

295 # Initialize the observed PSFs 

296 if not isinstance(psfCenter, Point2D): 

297 psfCenter = Point2D(*psfCenter) 

298 psfModels = mExposure.computePsfKernelImage(psfCenter) 

299 

300 # Use the inverse variance as the weights 

301 if useWeights: 

302 weights = 1/mExposure.variance.array 

303 else: 

304 # Mask out bad pixels 

305 weights = np.ones_like(mExposure.image.array) 

306 if badPixelMasks is None: 

307 badPixelMasks = ScarletDeblendConfig().badMask 

308 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks) 

309 mask = mExposure.mask.array & badPixels 

310 weights[mask > 0] = 0 

311 

312 if footprint is not None: 

313 # Mask out the pixels outside the footprint 

314 weights *= footprint.spans.asArray() 

315 

316 return lite.LiteObservation( 

317 images=mExposure.image.array, 

318 variance=mExposure.variance.array, 

319 weights=weights, 

320 psfs=psfModels, 

321 model_psf=modelPsf[None, :, :], 

322 convolution_mode=convolutionType, 

323 ) 

324 

325 

326def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None): 

327 """Deblend a parent footprint 

328 

329 Parameters 

330 ---------- 

331 mExposure : `lsst.image.MultibandExposure` 

332 - The multiband exposure containing the image, 

333 mask, and variance data 

334 footprint : `lsst.detection.Footprint` 

335 - The footprint of the parent to deblend 

336 config : `ScarletDeblendConfig` 

337 - Configuration of the deblending task 

338 """ 

339 # Extract coordinates from each MultiColorPeak 

340 bbox = footprint.getBBox() 

341 psfCenter = footprint.getCentroid() 

342 

343 observation = buildLiteObservation( 

344 modelPsf=modelPsf, 

345 psfCenter=psfCenter, 

346 mExposure=mExposure[:, bbox], 

347 footprint=footprint, 

348 badPixelMasks=config.badMask, 

349 useWeights=config.useWeights, 

350 convolutionType=config.convolutionType, 

351 ) 

352 

353 # Convert the centers to pixel coordinates 

354 xmin = bbox.getMinX() 

355 ymin = bbox.getMinY() 

356 centers = [ 

357 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

358 for peak in footprint.peaks 

359 if not isPseudoSource(peak, config.pseudoColumns) 

360 ] 

361 

362 # Initialize the sources 

363 if config.morphImage == "chi2": 

364 sources = lite.init_all_sources_main( 

365 observation, 

366 centers, 

367 min_snr=config.minSNR, 

368 thresh=config.morphThresh, 

369 ) 

370 elif config.morphImage == "wavelet": 

371 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin()) 

372 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)] 

373 sources = lite.init_all_sources_wavelets( 

374 observation, 

375 centers, 

376 use_psf=False, 

377 wavelets=_wavelets, 

378 min_snr=config.minSNR, 

379 ) 

380 else: 

381 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.") 

382 

383 # Set the optimizer 

384 if config.optimizer == "adaprox": 

385 parameterization = partial( 

386 lite.init_adaprox_component, 

387 bg_thresh=config.backgroundThresh, 

388 max_prox_iter=config.maxProxIter, 

389 ) 

390 elif config.optimizer == "fista": 

391 parameterization = partial( 

392 lite.init_fista_component, 

393 bg_thresh=config.backgroundThresh, 

394 ) 

395 else: 

396 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.") 

397 sources = lite.parameterize_sources(sources, observation, parameterization) 

398 

399 # Attach the peak to all of the initialized sources 

400 for k, center in enumerate(centers): 

401 # This is just to make sure that there isn't a coding bug 

402 if len(sources[k].components) > 0 and np.any(sources[k].center != center): 

403 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}") 

404 # Store the record for the peak with the appropriate source 

405 sources[k].detectedPeak = footprint.peaks[k] 

406 

407 blend = lite.LiteBlend(sources, observation) 

408 

409 # Initialize each source with its best fit spectrum 

410 if spectrumInit: 

411 blend.fit_spectra() 

412 

413 # Set the sources that could not be initialized and were skipped 

414 skipped = [src for src in sources if src.is_null] 

415 

416 blend.fit( 

417 max_iter=config.maxIter, 

418 e_rel=config.relativeError, 

419 min_iter=config.minIter, 

420 reweight=False, 

421 ) 

422 

423 # Store the location of the PSF center for storage 

424 blend.psfCenter = (psfCenter.x, psfCenter.y) 

425 

426 return blend, skipped 

427 

428 

429@dataclass 

430class DeblenderMetrics: 

431 """Metrics and measurements made on single sources. 

432 

433 Store deblender metrics to be added as attributes to a scarlet source 

434 before it is converted into a `SourceRecord`. 

435 When DM-34414 is finished this class will be eliminated and the metrics 

436 will be added to the schema using a pipeline task that calculates them 

437 from the stored deconvolved models. 

438 

439 All of the parameters are one dimensional numpy arrays, 

440 with an element for each band in the observed images. 

441 

442 `maxOverlap` is useful as a metric for determining how blended a source 

443 is because if it only overlaps with other sources at or below 

444 the noise level, it is likely to be a mostly isolated source 

445 in the deconvolved model frame. 

446 

447 `fluxOverlapFraction` is potentially more useful than the canonical 

448 "blendedness" (or purity) metric because it accounts for potential 

449 biases created during deblending by not weighting the overlapping 

450 flux with the flux of this sources model. 

451 

452 Attributes 

453 ---------- 

454 maxOverlap : `numpy.ndarray` 

455 The maximum overlap that the source has with its neighbors in 

456 a single pixel. 

457 fluxOverlap : `numpy.ndarray` 

458 The total flux from neighbors overlapping with the current source. 

459 fluxOverlapFraction : `numpy.ndarray` 

460 The fraction of `flux from neighbors/source flux` for a 

461 given source within the source's footprint. 

462 blendedness : `numpy.ndarray` 

463 The metric for determining how blended a source is using the 

464 Bosch et al. 2018 metric for "blendedness." Note that some 

465 surveys use the term "purity," which is `1-blendedness`. 

466 """ 

467 maxOverlap: np.array 

468 fluxOverlap: np.array 

469 fluxOverlapFraction: np.array 

470 blendedness: np.array 

471 

472 

473def setDeblenderMetrics(blend): 

474 """Set metrics that can be used to evalute the deblender accuracy 

475 

476 This function calculates the `DeblenderMetrics` for each source in the 

477 blend, and assigns it to that sources `metrics` property in place. 

478 

479 Parameters 

480 ---------- 

481 blend : `scarlet.lite.Blend` 

482 The blend containing the sources to measure. 

483 """ 

484 # Store the full model of the scene for comparison 

485 blendModel = blend.get_model() 

486 for k, src in enumerate(blend.sources): 

487 # Extract the source model in the full bounding box 

488 model = src.get_model(bbox=blend.bbox) 

489 # The footprint is the 2D array of non-zero pixels in each band 

490 footprint = np.bitwise_or.reduce(model > 0, axis=0) 

491 # Calculate the metrics. 

492 # See `DeblenderMetrics` for a description of each metric. 

493 neighborOverlap = (blendModel-model) * footprint[None, :, :] 

494 maxOverlap = np.max(neighborOverlap, axis=(1, 2)) 

495 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2)) 

496 fluxModel = np.sum(model, axis=(1, 2)) 

497 fluxOverlapFraction = np.zeros((len(model), ), dtype=float) 

498 isFinite = fluxModel > 0 

499 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite] 

500 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2)) 

501 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness) 

502 

503 

504class ScarletDeblendConfig(pexConfig.Config): 

505 """MultibandDeblendConfig 

506 

507 Configuration for the multiband deblender. 

508 The parameters are organized by the parameter types, which are 

509 - Stopping Criteria: Used to determine if the fit has converged 

510 - Position Fitting Criteria: Used to fit the positions of the peaks 

511 - Constraints: Used to apply constraints to the peaks and their components 

512 - Other: Parameters that don't fit into the above categories 

513 """ 

514 # Stopping Criteria 

515 minIter = pexConfig.Field(dtype=int, default=1, 

516 doc="Minimum number of iterations before the optimizer is allowed to stop.") 

517 maxIter = pexConfig.Field(dtype=int, default=300, 

518 doc=("Maximum number of iterations to deblend a single parent")) 

519 relativeError = pexConfig.Field(dtype=float, default=1e-2, 

520 doc=("Change in the loss function between iterations to exit fitter. " 

521 "Typically this is `1e-2` if measurements will be made on the " 

522 "flux re-distributed models and `1e-4` when making measurements " 

523 "on the models themselves.")) 

524 

525 # Constraints 

526 morphThresh = pexConfig.Field(dtype=float, default=1, 

527 doc="Fraction of background RMS a pixel must have" 

528 "to be included in the initial morphology") 

529 # Lite Parameters 

530 # All of these parameters (except version) are only valid if version='lite' 

531 version = pexConfig.ChoiceField( 

532 dtype=str, 

533 default="lite", 

534 allowed={ 

535 "scarlet": "main scarlet version (likely to be deprecated soon)", 

536 "lite": "Optimized version of scarlet for survey data from a single instrument", 

537 }, 

538 doc="The version of scarlet to use.", 

539 ) 

540 optimizer = pexConfig.ChoiceField( 

541 dtype=str, 

542 default="adaprox", 

543 allowed={ 

544 "adaprox": "Proximal ADAM optimization", 

545 "fista": "Accelerated proximal gradient method", 

546 }, 

547 doc="The optimizer to use for fitting parameters and is only used when version='lite'", 

548 ) 

549 morphImage = pexConfig.ChoiceField( 

550 dtype=str, 

551 default="chi2", 

552 allowed={ 

553 "chi2": "Initialize sources on a chi^2 image made from all available bands", 

554 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image", 

555 }, 

556 doc="The type of image to use for initializing the morphology. " 

557 "Must be either 'chi2' or 'wavelet'. " 

558 ) 

559 backgroundThresh = pexConfig.Field( 

560 dtype=float, 

561 default=0.25, 

562 doc="Fraction of background to use for a sparsity threshold. " 

563 "This prevents sources from growing unrealistically outside " 

564 "the parent footprint while still modeling flux correctly " 

565 "for bright sources." 

566 ) 

567 maxProxIter = pexConfig.Field( 

568 dtype=int, 

569 default=1, 

570 doc="Maximum number of proximal operator iterations inside of each " 

571 "iteration of the optimizer. " 

572 "This config field is only used if version='lite' and optimizer='adaprox'." 

573 ) 

574 waveletScales = pexConfig.Field( 

575 dtype=int, 

576 default=5, 

577 doc="Number of wavelet scales to use for wavelet initialization. " 

578 "This field is only used when `version`='lite' and `morphImage`='wavelet'." 

579 ) 

580 

581 # Other scarlet paremeters 

582 useWeights = pexConfig.Field( 

583 dtype=bool, default=True, 

584 doc=("Whether or not use use inverse variance weighting." 

585 "If `useWeights` is `False` then flat weights are used")) 

586 modelPsfSize = pexConfig.Field( 

587 dtype=int, default=11, 

588 doc="Model PSF side length in pixels") 

589 modelPsfSigma = pexConfig.Field( 

590 dtype=float, default=0.8, 

591 doc="Define sigma for the model frame PSF") 

592 minSNR = pexConfig.Field( 

593 dtype=float, default=50, 

594 doc="Minimum Signal to noise to accept the source." 

595 "Sources with lower flux will be initialized with the PSF but updated " 

596 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).") 

597 saveTemplates = pexConfig.Field( 

598 dtype=bool, default=True, 

599 doc="Whether or not to save the SEDs and templates") 

600 processSingles = pexConfig.Field( 

601 dtype=bool, default=True, 

602 doc="Whether or not to process isolated sources in the deblender") 

603 convolutionType = pexConfig.Field( 

604 dtype=str, default="fft", 

605 doc="Type of convolution to render the model to the observations.\n" 

606 "- 'fft': perform convolutions in Fourier space\n" 

607 "- 'real': peform convolutions in real space.") 

608 sourceModel = pexConfig.Field( 

609 dtype=str, default="double", 

610 doc=("How to determine which model to use for sources, from\n" 

611 "- 'single': use a single component for all sources\n" 

612 "- 'double': use a bulge disk model for all sources\n" 

613 "- 'compact': use a single component model, initialzed with a point source morphology, " 

614 " for all sources\n" 

615 "- 'point': use a point-source model for all sources\n" 

616 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"), 

617 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.", 

618 ) 

619 setSpectra = pexConfig.Field( 

620 dtype=bool, default=True, 

621 doc="Whether or not to solve for the best-fit spectra during initialization. " 

622 "This makes initialization slightly longer, as it requires a convolution " 

623 "to set the optimal spectra, but results in a much better initial log-likelihood " 

624 "and reduced total runtime, with convergence in fewer iterations." 

625 "This option is only used when " 

626 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.") 

627 

628 # Mask-plane restrictions 

629 badMask = pexConfig.ListField( 

630 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"], 

631 doc="Whether or not to process isolated sources in the deblender") 

632 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

633 doc="Mask planes to ignore when performing statistics") 

634 maskLimits = pexConfig.DictField( 

635 keytype=str, 

636 itemtype=float, 

637 default={}, 

638 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

639 "Sources violating this limit will not be deblended. " 

640 "If the fraction is `0` then the limit is a single pixel."), 

641 ) 

642 

643 # Size restrictions 

644 maxNumberOfPeaks = pexConfig.Field( 

645 dtype=int, default=200, 

646 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

647 " (<= 0: unlimited)")) 

648 maxFootprintArea = pexConfig.Field( 

649 dtype=int, default=100_000, 

650 doc=("Maximum area for footprints before they are ignored as large; " 

651 "non-positive means no threshold applied")) 

652 maxAreaTimesPeaks = pexConfig.Field( 

653 dtype=int, default=10_000_000, 

654 doc=("Maximum rectangular footprint area * nPeaks in the footprint. " 

655 "This was introduced in DM-33690 to prevent fields that are crowded or have a " 

656 "LSB galaxy that causes memory intensive initialization in scarlet from dominating " 

657 "the overall runtime and/or causing the task to run out of memory. " 

658 "(<= 0: unlimited)") 

659 ) 

660 maxFootprintSize = pexConfig.Field( 

661 dtype=int, default=0, 

662 doc=("Maximum linear dimension for footprints before they are ignored " 

663 "as large; non-positive means no threshold applied")) 

664 minFootprintAxisRatio = pexConfig.Field( 

665 dtype=float, default=0.0, 

666 doc=("Minimum axis ratio for footprints before they are ignored " 

667 "as large; non-positive means no threshold applied")) 

668 maxSpectrumCutoff = pexConfig.Field( 

669 dtype=int, default=1_000_000, 

670 doc=("Maximum number of pixels * number of sources in a blend. " 

671 "This is different than `maxFootprintArea` because this isn't " 

672 "the footprint area but the area of the bounding box that " 

673 "contains the footprint, and is also multiplied by the number of" 

674 "sources in the footprint. This prevents large skinny blends with " 

675 "a high density of sources from running out of memory. " 

676 "If `maxSpectrumCutoff == -1` then there is no cutoff.") 

677 ) 

678 # Failure modes 

679 fallback = pexConfig.Field( 

680 dtype=bool, default=True, 

681 doc="Whether or not to fallback to a smaller number of components if a source does not initialize" 

682 ) 

683 notDeblendedMask = pexConfig.Field( 

684 dtype=str, default="NOT_DEBLENDED", optional=True, 

685 doc="Mask name for footprints not deblended, or None") 

686 catchFailures = pexConfig.Field( 

687 dtype=bool, default=True, 

688 doc=("If True, catch exceptions thrown by the deblender, log them, " 

689 "and set a flag on the parent, instead of letting them propagate up")) 

690 

691 # Other options 

692 columnInheritance = pexConfig.DictField( 

693 keytype=str, itemtype=str, default={ 

694 "deblend_nChild": "deblend_parentNChild", 

695 "deblend_nPeaks": "deblend_parentNPeaks", 

696 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag", 

697 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag", 

698 }, 

699 doc="Columns to pass from the parent to the child. " 

700 "The key is the name of the column for the parent record, " 

701 "the value is the name of the column to use for the child." 

702 ) 

703 pseudoColumns = pexConfig.ListField( 

704 dtype=str, default=['merge_peak_sky', 'sky_source'], 

705 doc="Names of flags which should never be deblended." 

706 ) 

707 

708 # Logging option(s) 

709 loggingInterval = pexConfig.Field( 

710 dtype=int, default=600, 

711 doc="Interval (in seconds) to log messages (at VERBOSE level) while deblending sources.", 

712 deprecated="This field is no longer used and will be removed in v25.", 

713 ) 

714 # Testing options 

715 # Some obs packages and ci packages run the full pipeline on a small 

716 # subset of data to test that the pipeline is functioning properly. 

717 # This is not meant as scientific validation, so it can be useful 

718 # to only run on a small subset of the data that is large enough to 

719 # test the desired pipeline features but not so long that the deblender 

720 # is the tall pole in terms of execution times. 

721 useCiLimits = pexConfig.Field( 

722 dtype=bool, default=False, 

723 doc="Limit the number of sources deblended for CI to prevent long build times") 

724 ciDeblendChildRange = pexConfig.ListField( 

725 dtype=int, default=[5, 10], 

726 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated." 

727 "If `useCiLimits==False` then this parameter is ignored.") 

728 ciNumParentsToDeblend = pexConfig.Field( 

729 dtype=int, default=10, 

730 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count " 

731 "within `ciDebledChildRange`. " 

732 "If `useCiLimits==False` then this parameter is ignored.") 

733 

734 

735class ScarletDeblendTask(pipeBase.Task): 

736 """ScarletDeblendTask 

737 

738 Split blended sources into individual sources. 

739 

740 This task has no return value; it only modifies the SourceCatalog in-place. 

741 """ 

742 ConfigClass = ScarletDeblendConfig 

743 _DefaultName = "scarletDeblend" 

744 

745 def __init__(self, schema, peakSchema=None, **kwargs): 

746 """Create the task, adding necessary fields to the given schema. 

747 

748 Parameters 

749 ---------- 

750 schema : `lsst.afw.table.schema.schema.Schema` 

751 Schema object for measurement fields; will be modified in-place. 

752 peakSchema : `lsst.afw.table.schema.schema.Schema` 

753 Schema of Footprint Peaks that will be passed to the deblender. 

754 Any fields beyond the PeakTable minimal schema will be transferred 

755 to the main source Schema. If None, no fields will be transferred 

756 from the Peaks. 

757 filters : list of str 

758 Names of the filters used for the eposures. This is needed to store 

759 the SED as a field 

760 **kwargs 

761 Passed to Task.__init__. 

762 """ 

763 pipeBase.Task.__init__(self, **kwargs) 

764 

765 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

766 if peakSchema is None: 

767 # In this case, the peakSchemaMapper will transfer nothing, but 

768 # we'll still have one 

769 # to simplify downstream code 

770 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

771 else: 

772 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

773 for item in peakSchema: 

774 if item.key not in peakMinimalSchema: 

775 self.peakSchemaMapper.addMapping(item.key, item.field) 

776 # Because SchemaMapper makes a copy of the output schema 

777 # you give its ctor, it isn't updating this Schema in 

778 # place. That's probably a design flaw, but in the 

779 # meantime, we'll keep that schema in sync with the 

780 # peakSchemaMapper.getOutputSchema() manually, by adding 

781 # the same fields to both. 

782 schema.addField(item.field) 

783 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

784 self._addSchemaKeys(schema) 

785 self.schema = schema 

786 self.toCopyFromParent = [item.key for item in self.schema 

787 if item.field.getName().startswith("merge_footprint")] 

788 

789 def _addSchemaKeys(self, schema): 

790 """Add deblender specific keys to the schema 

791 """ 

792 # Parent (blend) fields 

793 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms') 

794 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge') 

795 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

796 doc='Number of children this object has (defaults to 0)') 

797 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32, 

798 doc="Number of initial peaks in the blend. " 

799 "This includes peaks that may have been culled " 

800 "during deblending or failed to deblend") 

801 # Skipped flags 

802 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

803 doc="Deblender skipped this source") 

804 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag', 

805 doc='The source has only a single peak ' 

806 'and was not deblended') 

807 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag', 

808 doc='The source is identified as a "pseudo" source and ' 

809 'was not deblended') 

810 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

811 doc='Source had too many peaks; ' 

812 'only the brightest were included') 

813 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

814 doc='Parent footprint covered too many pixels') 

815 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

816 doc='Parent footprint had too many masked pixels') 

817 # Convergence flags 

818 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag', 

819 doc='scarlet sed optimization did not converge before' 

820 'config.maxIter') 

821 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag', 

822 doc='scarlet morph optimization did not converge before' 

823 'config.maxIter') 

824 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag', 

825 type='Flag', 

826 doc='at least one source in the blend' 

827 'failed to converge') 

828 # Error flags 

829 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

830 doc="Deblending failed on source") 

831 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25, 

832 doc='Name of error if the blend failed') 

833 # Deblended source fields 

834 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center", 

835 doc="Center used to apply constraints in scarlet", 

836 unit="pixel") 

837 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32, 

838 doc="ID of the peak in the parent footprint. " 

839 "This is not unique, but the combination of 'parent'" 

840 "and 'peakId' should be for all child sources. " 

841 "Top level blends with no parents have 'peakId=0'") 

842 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count', 

843 doc="The instFlux at the peak position of deblended mode") 

844 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25, 

845 doc="The type of model used, for example " 

846 "MultiExtendedSource, SingleExtendedSource, PointSource") 

847 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32, 

848 doc="deblend_nPeaks from this records parent.") 

849 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32, 

850 doc="deblend_nChild from this records parent.") 

851 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32, 

852 doc="Flux measurement from scarlet") 

853 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32, 

854 doc="Final logL, used to identify regressions in scarlet.") 

855 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag', 

856 doc='Source had flux on the edge of the parent footprint') 

857 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag', 

858 doc="True when scarlet initializes sources " 

859 "in the blend with a more accurate spectrum. " 

860 "The algorithm uses a lot of memory, " 

861 "so large dense blends will use " 

862 "a less accurate initialization.") 

863 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32, 

864 doc="Number of components in a ScarletLiteSource. " 

865 "If `config.version != 'lite'`then " 

866 "this column is set to zero.") 

867 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

868 doc='Deblender thought this source looked like a PSF') 

869 # Blendedness/classification metrics 

870 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32, 

871 doc="Maximum overlap with all of the other neighbors flux " 

872 "combined." 

873 "This is useful as a metric for determining how blended a " 

874 "source is because if it only overlaps with other sources " 

875 "at or below the noise level, it is likely to be a mostly " 

876 "isolated source in the deconvolved model frame.") 

877 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32, 

878 doc="This is the total flux from neighboring objects that " 

879 "overlaps with this source.") 

880 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32, 

881 doc="This is the fraction of " 

882 "`flux from neighbors/source flux` " 

883 "for a given source within the source's" 

884 "footprint.") 

885 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32, 

886 doc="The Bosch et al. 2018 metric for 'blendedness.' ") 

887 

888 @timeMethod 

889 def run(self, mExposure, mergedSources): 

890 """Get the psf from each exposure and then run deblend(). 

891 

892 Parameters 

893 ---------- 

894 mExposure : `MultibandExposure` 

895 The exposures should be co-added images of the same 

896 shape and region of the sky. 

897 mergedSources : `SourceCatalog` 

898 The merged `SourceCatalog` that contains parent footprints 

899 to (potentially) deblend. 

900 

901 Returns 

902 ------- 

903 templateCatalogs: dict 

904 Keys are the names of the filters and the values are 

905 `lsst.afw.table.source.source.SourceCatalog`'s. 

906 These are catalogs with heavy footprints that are the templates 

907 created by the multiband templates. 

908 """ 

909 return self.deblend(mExposure, mergedSources) 

910 

911 @timeMethod 

912 def deblend(self, mExposure, catalog): 

913 """Deblend a data cube of multiband images 

914 

915 Parameters 

916 ---------- 

917 mExposure : `MultibandExposure` 

918 The exposures should be co-added images of the same 

919 shape and region of the sky. 

920 catalog : `SourceCatalog` 

921 The merged `SourceCatalog` that contains parent footprints 

922 to (potentially) deblend. The new deblended sources are 

923 appended to this catalog in place. 

924 

925 Returns 

926 ------- 

927 catalogs : `dict` or `None` 

928 Keys are the names of the filters and the values are 

929 `lsst.afw.table.source.source.SourceCatalog`'s. 

930 These are catalogs with heavy footprints that are the templates 

931 created by the multiband templates. 

932 """ 

933 import time 

934 

935 # Cull footprints if required by ci 

936 if self.config.useCiLimits: 

937 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.", 

938 len(catalog)) 

939 # Select parents with a number of children in the range 

940 # config.ciDeblendChildRange 

941 minChildren, maxChildren = self.config.ciDeblendChildRange 

942 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog]) 

943 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0] 

944 if len(childrenInRange) < self.config.ciNumParentsToDeblend: 

945 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range " 

946 "indicated by ciDeblendChildRange. Adjust this range to include more " 

947 "parents.") 

948 # Keep all of the isolated parents and the first 

949 # `ciNumParentsToDeblend` children 

950 parents = nPeaks == 1 

951 children = np.zeros((len(catalog),), dtype=bool) 

952 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True 

953 catalog = catalog[parents | children] 

954 # We need to update the IdFactory, otherwise the the source ids 

955 # will not be sequential 

956 idFactory = catalog.getIdFactory() 

957 maxId = np.max(catalog["id"]) 

958 idFactory.notify(maxId) 

959 

960 filters = mExposure.filters 

961 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure)) 

962 periodicLog = PeriodicLogger(self.log) 

963 

964 # Create a set of wavelet coefficients if using wavelet initialization 

965 if self.config.version == "lite" and self.config.morphImage == "wavelet": 

966 images = mExposure.image.array 

967 variance = mExposure.variance.array 

968 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales) 

969 else: 

970 wavelets = None 

971 

972 # Add the NOT_DEBLENDED mask to the mask plane in each band 

973 if self.config.notDeblendedMask: 

974 for mask in mExposure.mask: 

975 mask.addMaskPlane(self.config.notDeblendedMask) 

976 

977 # Initialize the persistable data model 

978 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma) 

979 dataModel = ScarletModelData(filters, modelPsf) 

980 

981 nParents = len(catalog) 

982 nDeblendedParents = 0 

983 skippedParents = [] 

984 for parentIndex in range(nParents): 

985 parent = catalog[parentIndex] 

986 foot = parent.getFootprint() 

987 bbox = foot.getBBox() 

988 peaks = foot.getPeaks() 

989 

990 # Since we use the first peak for the parent object, we should 

991 # propagate its flags to the parent source. 

992 parent.assign(peaks[0], self.peakSchemaMapper) 

993 

994 # Block of conditions for skipping a parent with multiple children 

995 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None: 

996 self._skipParent(parent, *skipArgs) 

997 skippedParents.append(parentIndex) 

998 continue 

999 

1000 nDeblendedParents += 1 

1001 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks)) 

1002 # Run the deblender 

1003 blendError = None 

1004 

1005 # Choose whether or not to use improved spectral initialization. 

1006 # This significantly cuts down on the number of iterations 

1007 # that the optimizer needs and usually results in a better 

1008 # fit. 

1009 # But using least squares on a very large blend causes memory 

1010 # issues, so it is not done for large blends 

1011 if self.config.setSpectra: 

1012 if self.config.maxSpectrumCutoff <= 0: 

1013 spectrumInit = True 

1014 else: 

1015 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff 

1016 else: 

1017 spectrumInit = False 

1018 

1019 try: 

1020 t0 = time.monotonic() 

1021 # Build the parameter lists with the same ordering 

1022 if self.config.version == "scarlet": 

1023 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit) 

1024 elif self.config.version == "lite": 

1025 blend, skipped = deblend_lite( 

1026 mExposure=mExposure, 

1027 modelPsf=modelPsf, 

1028 footprint=foot, 

1029 config=self.config, 

1030 spectrumInit=spectrumInit, 

1031 wavelets=wavelets, 

1032 ) 

1033 tf = time.monotonic() 

1034 runtime = (tf-t0)*1000 

1035 converged = _checkBlendConvergence(blend, self.config.relativeError) 

1036 # Store the number of components in the blend 

1037 if self.config.version == "lite": 

1038 nComponents = len(blend.components) 

1039 else: 

1040 nComponents = 0 

1041 nChild = len(blend.sources) 

1042 # Catch all errors and filter out the ones that we know about 

1043 except Exception as e: 

1044 print("deblend failed") 

1045 print(e) 

1046 blendError = type(e).__name__ 

1047 if isinstance(e, ScarletGradientError): 

1048 parent.set(self.iterKey, e.iterations) 

1049 elif not isinstance(e, IncompleteDataError): 

1050 blendError = "UnknownError" 

1051 if self.config.catchFailures: 

1052 # Make it easy to find UnknownErrors in the log file 

1053 self.log.warn("UnknownError") 

1054 import traceback 

1055 traceback.print_exc() 

1056 else: 

1057 raise 

1058 

1059 self._skipParent( 

1060 parent=parent, 

1061 skipKey=self.deblendFailedKey, 

1062 logMessage=f"Unable to deblend source {parent.getId}: {blendError}", 

1063 ) 

1064 parent.set(self.deblendErrorKey, blendError) 

1065 skippedParents.append(parentIndex) 

1066 continue 

1067 

1068 # Update the parent record with the deblending results 

1069 if self.config.version == "scarlet": 

1070 logL = -blend.loss[-1] + blend.observations[0].log_norm 

1071 elif self.config.version == "lite": 

1072 logL = blend.loss[-1] 

1073 self._updateParentRecord( 

1074 parent=parent, 

1075 nPeaks=len(peaks), 

1076 nChild=nChild, 

1077 nComponents=nComponents, 

1078 runtime=runtime, 

1079 iterations=len(blend.loss), 

1080 logL=logL, 

1081 spectrumInit=spectrumInit, 

1082 converged=converged, 

1083 ) 

1084 

1085 # Add each deblended source to the catalog 

1086 for k, scarletSource in enumerate(blend.sources): 

1087 # Skip any sources with no flux or that scarlet skipped because 

1088 # it could not initialize 

1089 if k in skipped or (self.config.version == "lite" and scarletSource.is_null): 

1090 # No need to propagate anything 

1091 continue 

1092 parent.set(self.deblendSkippedKey, False) 

1093 

1094 # Add all fields except the HeavyFootprint to the 

1095 # source record 

1096 sourceRecord = self._addChild( 

1097 parent=parent, 

1098 peak=scarletSource.detectedPeak, 

1099 catalog=catalog, 

1100 scarletSource=scarletSource, 

1101 ) 

1102 scarletSource.recordId = sourceRecord.getId() 

1103 scarletSource.peakId = scarletSource.detectedPeak.getId() 

1104 

1105 # Store the blend information so that it can be persisted 

1106 if self.config.version == "lite": 

1107 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin()) 

1108 else: 

1109 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin()) 

1110 dataModel.blends[parent.getId()] = blendData 

1111 

1112 # Log a message if it has been a while since the last log. 

1113 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents) 

1114 

1115 # Clear the cached values in scarlet to clear out memory 

1116 scarlet.cache.Cache._cache = {} 

1117 

1118 # Update the mExposure mask with the footprint of skipped parents 

1119 if self.config.notDeblendedMask: 

1120 for mask in mExposure.mask: 

1121 for parentIndex in skippedParents: 

1122 fp = catalog[parentIndex].getFootprint() 

1123 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask)) 

1124 

1125 self.log.info("Deblender results: of %d parent sources, %d were deblended, " 

1126 "creating %d children, for a total of %d sources", 

1127 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog)) 

1128 return catalog, dataModel 

1129 

1130 def _isLargeFootprint(self, footprint): 

1131 """Returns whether a Footprint is large 

1132 

1133 'Large' is defined by thresholds on the area, size and axis ratio, 

1134 and total area of the bounding box multiplied by 

1135 the number of children. 

1136 These may be disabled independently by configuring them to be 

1137 non-positive. 

1138 """ 

1139 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

1140 return True 

1141 if self.config.maxFootprintSize > 0: 

1142 bbox = footprint.getBBox() 

1143 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

1144 return True 

1145 if self.config.minFootprintAxisRatio > 0: 

1146 axes = afwEll.Axes(footprint.getShape()) 

1147 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

1148 return True 

1149 if self.config.maxAreaTimesPeaks > 0: 

1150 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks: 

1151 return True 

1152 return False 

1153 

1154 def _isMasked(self, footprint, mExposure): 

1155 """Returns whether the footprint violates the mask limits 

1156 

1157 Parameters 

1158 ---------- 

1159 footprint : `lsst.afw.detection.Footprint` 

1160 The footprint to check for masked pixels 

1161 mMask : `lsst.afw.image.MaskX` 

1162 The mask plane to check for masked pixels in the `footprint`. 

1163 

1164 Returns 

1165 ------- 

1166 isMasked : `bool` 

1167 `True` if `self.config.maskPlaneLimits` is less than the 

1168 fraction of pixels for a given mask in 

1169 `self.config.maskLimits`. 

1170 """ 

1171 bbox = footprint.getBBox() 

1172 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0) 

1173 size = float(footprint.getArea()) 

1174 for maskName, limit in self.config.maskLimits.items(): 

1175 maskVal = mExposure.mask.getPlaneBitMask(maskName) 

1176 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin()) 

1177 # spanset of masked pixels 

1178 maskedSpan = footprint.spans.intersect(_mask, maskVal) 

1179 if (maskedSpan.getArea())/size > limit: 

1180 return True 

1181 return False 

1182 

1183 def _skipParent(self, parent, skipKey, logMessage): 

1184 """Update a parent record that is not being deblended. 

1185 

1186 This is a fairly trivial function but is implemented to ensure 

1187 that a skipped parent updates the appropriate columns 

1188 consistently, and always has a flag to mark the reason that 

1189 it is being skipped. 

1190 

1191 Parameters 

1192 ---------- 

1193 parent : `lsst.afw.table.source.source.SourceRecord` 

1194 The parent record to flag as skipped. 

1195 skipKey : `bool` 

1196 The name of the flag to mark the reason for skipping. 

1197 logMessage : `str` 

1198 The message to display in a log.trace when a source 

1199 is skipped. 

1200 """ 

1201 if logMessage is not None: 

1202 self.log.trace(logMessage) 

1203 self._updateParentRecord( 

1204 parent=parent, 

1205 nPeaks=len(parent.getFootprint().peaks), 

1206 nChild=0, 

1207 nComponents=0, 

1208 runtime=np.nan, 

1209 iterations=0, 

1210 logL=np.nan, 

1211 spectrumInit=False, 

1212 converged=False, 

1213 ) 

1214 

1215 # Mark the source as skipped by the deblender and 

1216 # flag the reason why. 

1217 parent.set(self.deblendSkippedKey, True) 

1218 parent.set(skipKey, True) 

1219 

1220 def _checkSkipped(self, parent, mExposure): 

1221 """Update a parent record that is not being deblended. 

1222 

1223 This is a fairly trivial function but is implemented to ensure 

1224 that a skipped parent updates the appropriate columns 

1225 consistently, and always has a flag to mark the reason that 

1226 it is being skipped. 

1227 

1228 Parameters 

1229 ---------- 

1230 parent : `lsst.afw.table.source.source.SourceRecord` 

1231 The parent record to flag as skipped. 

1232 mExposure : `MultibandExposure` 

1233 The exposures should be co-added images of the same 

1234 shape and region of the sky. 

1235 Returns 

1236 ------- 

1237 skip: `bool` 

1238 `True` if the deblender will skip the parent 

1239 """ 

1240 skipKey = None 

1241 skipMessage = None 

1242 footprint = parent.getFootprint() 

1243 if len(footprint.peaks) < 2 and not self.config.processSingles: 

1244 # Skip isolated sources unless processSingles is turned on. 

1245 # Note: this does not flag isolated sources as skipped or 

1246 # set the NOT_DEBLENDED mask in the exposure, 

1247 # since these aren't really any skipped blends. 

1248 skipKey = self.isolatedParentKey 

1249 elif isPseudoSource(parent, self.config.pseudoColumns): 

1250 # We also skip pseudo sources, like sky objects, which 

1251 # are intended to be skipped. 

1252 skipKey = self.pseudoKey 

1253 if self._isLargeFootprint(footprint): 

1254 # The footprint is above the maximum footprint size limit 

1255 skipKey = self.tooBigKey 

1256 skipMessage = f"Parent {parent.getId()}: skipping large footprint" 

1257 elif self._isMasked(footprint, mExposure): 

1258 # The footprint exceeds the maximum number of masked pixels 

1259 skipKey = self.maskedKey 

1260 skipMessage = f"Parent {parent.getId()}: skipping masked footprint" 

1261 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks: 

1262 # Unlike meas_deblender, in scarlet we skip the entire blend 

1263 # if the number of peaks exceeds max peaks, since neglecting 

1264 # to model any peaks often results in catastrophic failure 

1265 # of scarlet to generate models for the brighter sources. 

1266 skipKey = self.tooManyPeaksKey 

1267 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks" 

1268 if skipKey is not None: 

1269 return (skipKey, skipMessage) 

1270 return None 

1271 

1272 def setSkipFlags(self, mExposure, catalog): 

1273 """Set the skip flags for all of the parent sources 

1274 

1275 This is mostly used for testing which parent sources will be deblended 

1276 and which will be skipped based on the current configuration options. 

1277 Skipped sources will have the appropriate flags set in place in the 

1278 catalog. 

1279 

1280 Parameters 

1281 ---------- 

1282 mExposure : `MultibandExposure` 

1283 The exposures should be co-added images of the same 

1284 shape and region of the sky. 

1285 catalog : `SourceCatalog` 

1286 The merged `SourceCatalog` that contains parent footprints 

1287 to (potentially) deblend. The new deblended sources are 

1288 appended to this catalog in place. 

1289 """ 

1290 for src in catalog: 

1291 if skipArgs := self._checkSkipped(src, mExposure) is not None: 

1292 self._skipParent(src, *skipArgs) 

1293 

1294 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents, 

1295 runtime, iterations, logL, spectrumInit, converged): 

1296 """Update a parent record in all of the single band catalogs. 

1297 

1298 Ensure that all locations that update a parent record, 

1299 whether it is skipped or updated after deblending, 

1300 update all of the appropriate columns. 

1301 

1302 Parameters 

1303 ---------- 

1304 parent : `lsst.afw.table.source.source.SourceRecord` 

1305 The parent record to update. 

1306 nPeaks : `int` 

1307 Number of peaks in the parent footprint. 

1308 nChild : `int` 

1309 Number of children deblended from the parent. 

1310 This may differ from `nPeaks` if some of the peaks 

1311 were culled and have no deblended model. 

1312 nComponents : `int` 

1313 Total number of components in the parent. 

1314 This is usually different than the number of children, 

1315 since it is common for a single source to have multiple 

1316 components. 

1317 runtime : `float` 

1318 Total runtime for deblending. 

1319 iterations : `int` 

1320 Total number of iterations in scarlet before convergence. 

1321 logL : `float` 

1322 Final log likelihood of the blend. 

1323 spectrumInit : `bool` 

1324 True when scarlet used `set_spectra` to initialize all 

1325 sources with better initial intensities. 

1326 converged : `bool` 

1327 True when the optimizer reached convergence before 

1328 reaching the maximum number of iterations. 

1329 """ 

1330 parent.set(self.nPeaksKey, nPeaks) 

1331 parent.set(self.nChildKey, nChild) 

1332 parent.set(self.nComponentsKey, nComponents) 

1333 parent.set(self.runtimeKey, runtime) 

1334 parent.set(self.iterKey, iterations) 

1335 parent.set(self.scarletLogLKey, logL) 

1336 parent.set(self.scarletSpectrumInitKey, spectrumInit) 

1337 parent.set(self.blendConvergenceFailedFlagKey, converged) 

1338 

1339 def _addChild(self, parent, peak, catalog, scarletSource): 

1340 """Add a child to a catalog. 

1341 

1342 This creates a new child in the source catalog, 

1343 assigning it a parent id, and adding all columns 

1344 that are independent across all filter bands. 

1345 

1346 Parameters 

1347 ---------- 

1348 parent : `lsst.afw.table.source.source.SourceRecord` 

1349 The parent of the new child record. 

1350 peak : `lsst.afw.table.PeakRecord` 

1351 The peak record for the peak from the parent peak catalog. 

1352 catalog : `lsst.afw.table.source.source.SourceCatalog` 

1353 The merged `SourceCatalog` that contains parent footprints 

1354 to (potentially) deblend. 

1355 scarletSource : `scarlet.Component` 

1356 The scarlet model for the new source record. 

1357 """ 

1358 src = catalog.addNew() 

1359 for key in self.toCopyFromParent: 

1360 src.set(key, parent.get(key)) 

1361 # The peak catalog is the same for all bands, 

1362 # so we just use the first peak catalog 

1363 src.assign(peak, self.peakSchemaMapper) 

1364 src.setParent(parent.getId()) 

1365 src.set(self.nPeaksKey, 1) 

1366 # Set the psf key based on whether or not the source was 

1367 # deblended using the PointSource model. 

1368 # This key is not that useful anymore since we now keep track of 

1369 # `modelType`, but we continue to propagate it in case code downstream 

1370 # is expecting it. 

1371 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource") 

1372 src.set(self.modelTypeKey, scarletSource.__class__.__name__) 

1373 # We set the runtime to zero so that summing up the 

1374 # runtime column will give the total time spent 

1375 # running the deblender for the catalog. 

1376 src.set(self.runtimeKey, 0) 

1377 

1378 # Set the position of the peak from the parent footprint 

1379 # This will make it easier to match the same source across 

1380 # deblenders and across observations, where the peak 

1381 # position is unlikely to change unless enough time passes 

1382 # for a source to move on the sky. 

1383 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"])) 

1384 src.set(self.peakIdKey, peak["id"]) 

1385 

1386 # Store the number of components for the source 

1387 src.set(self.nComponentsKey, len(scarletSource.components)) 

1388 

1389 # Propagate columns from the parent to the child 

1390 for parentColumn, childColumn in self.config.columnInheritance.items(): 

1391 src.set(childColumn, parent.get(parentColumn)) 

1392 

1393 return src