Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%

447 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-12 03:11 -0700

1# This file is part of meas_extensions_scarlet. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from dataclasses import dataclass 

23from functools import partial 

24import logging 

25import numpy as np 

26import scarlet 

27from scarlet.psf import ImagePSF, GaussianPSF 

28from scarlet import Blend, Frame, Observation 

29from scarlet.renderer import ConvolutionRenderer 

30from scarlet.detect import get_detect_wavelets 

31from scarlet.initialization import init_all_sources 

32from scarlet import lite 

33 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36from lsst.geom import Point2I, Point2D 

37import lsst.afw.geom.ellipses as afwEll 

38import lsst.afw.image as afwImage 

39import lsst.afw.detection as afwDet 

40import lsst.afw.table as afwTable 

41from lsst.utils.logging import PeriodicLogger 

42from lsst.utils.timer import timeMethod 

43 

44from .source import bboxToScarletBox 

45from .io import ScarletModelData, scarletToData, scarletLiteToData 

46 

47# Scarlet and proxmin have a different definition of log levels than the stack, 

48# so even "warnings" occur far more often than we would like. 

49# So for now we only display scarlet and proxmin errors, as all other 

50# scarlet outputs would be considered "TRACE" by our standards. 

51scarletLogger = logging.getLogger("scarlet") 

52scarletLogger.setLevel(logging.ERROR) 

53proxminLogger = logging.getLogger("proxmin") 

54proxminLogger.setLevel(logging.ERROR) 

55 

56__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"] 

57 

58logger = logging.getLogger(__name__) 

59 

60 

61class IncompleteDataError(Exception): 

62 """The PSF could not be computed due to incomplete data 

63 """ 

64 pass 

65 

66 

67class ScarletGradientError(Exception): 

68 """An error occurred during optimization 

69 

70 This error occurs when the optimizer encounters 

71 a NaN value while calculating the gradient. 

72 """ 

73 def __init__(self, iterations, sources): 

74 self.iterations = iterations 

75 self.sources = sources 

76 msg = ("ScalarGradientError in iteration {0}. " 

77 "NaN values introduced in sources {1}") 

78 self.message = msg.format(iterations, sources) 

79 

80 def __str__(self): 

81 return self.message 

82 

83 

84def _checkBlendConvergence(blend, f_rel): 

85 """Check whether or not a blend has converged 

86 """ 

87 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1]) 

88 convergence = f_rel * np.abs(blend.loss[-1]) 

89 return deltaLoss < convergence 

90 

91 

92def isPseudoSource(source, pseudoColumns): 

93 """Check if a source is a pseudo source. 

94 

95 This is mostly for skipping sky objects, 

96 but any other column can also be added to disable 

97 deblending on a parent or individual source when 

98 set to `True`. 

99 

100 Parameters 

101 ---------- 

102 source : `lsst.afw.table.source.source.SourceRecord` 

103 The source to check for the pseudo bit. 

104 pseudoColumns : `list` of `str` 

105 A list of columns to check for pseudo sources. 

106 """ 

107 isPseudo = False 

108 for col in pseudoColumns: 

109 try: 

110 isPseudo |= source[col] 

111 except KeyError: 

112 pass 

113 return isPseudo 

114 

115 

116def deblend(mExposure, footprint, config, spectrumInit): 

117 """Deblend a parent footprint 

118 

119 Parameters 

120 ---------- 

121 mExposure : `lsst.image.MultibandExposure` 

122 The multiband exposure containing the image, 

123 mask, and variance data. 

124 footprint : `lsst.detection.Footprint` 

125 The footprint of the parent to deblend. 

126 config : `ScarletDeblendConfig` 

127 Configuration of the deblending task. 

128 spectrumInit : `bool` 

129 Whether or not to initialize the spectrum. 

130 

131 Returns 

132 ------- 

133 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData` 

134 The persistable representation of a `scarlet.Blend`. 

135 skipped : `list` of `int` 

136 The indices of any children that failed to initialize 

137 and were skipped. 

138 spectrumInit : `bool` 

139 Whether or not all of the sources were initialized by jointly 

140 fitting their SED's. This provides a better initialization 

141 but created memory issues when a blend is too large or 

142 contains too many sources. 

143 """ 

144 # Extract coordinates from each MultiColorPeak 

145 bbox = footprint.getBBox() 

146 

147 # Create the data array from the masked images 

148 images = mExposure.image[:, bbox].array 

149 

150 # Use the inverse variance as the weights 

151 if config.useWeights: 

152 weights = 1/mExposure.variance[:, bbox].array 

153 else: 

154 weights = np.ones_like(images) 

155 badPixels = mExposure.mask.getPlaneBitMask(config.badMask) 

156 mask = mExposure.mask[:, bbox].array & badPixels 

157 weights[mask > 0] = 0 

158 

159 # Mask out the pixels outside the footprint 

160 weights *= footprint.spans.asArray() 

161 

162 psfCenter = footprint.getCentroid() 

163 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32) 

164 psfs = ImagePSF(psfs) 

165 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters)) 

166 

167 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters) 

168 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters) 

169 if config.convolutionType == "fft": 

170 observation.match(frame) 

171 elif config.convolutionType == "real": 

172 renderer = ConvolutionRenderer(observation, frame, convolution_type="real") 

173 observation.match(frame, renderer=renderer) 

174 else: 

175 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType)) 

176 

177 assert(config.sourceModel in ["single", "double", "compact", "fit"]) 

178 

179 # Set the appropriate number of components 

180 if config.sourceModel == "single": 

181 maxComponents = 1 

182 elif config.sourceModel == "double": 

183 maxComponents = 2 

184 elif config.sourceModel == "compact": 

185 maxComponents = 0 

186 elif config.sourceModel == "point": 

187 raise NotImplementedError("Point source photometry is currently not implemented") 

188 elif config.sourceModel == "fit": 

189 # It is likely in the future that there will be some heuristic 

190 # used to determine what type of model to use for each source, 

191 # but that has not yet been implemented (see DM-22551) 

192 raise NotImplementedError("sourceModel 'fit' has not been implemented yet") 

193 

194 # Convert the centers to pixel coordinates 

195 xmin = bbox.getMinX() 

196 ymin = bbox.getMinY() 

197 centers = [ 

198 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

199 for peak in footprint.peaks 

200 if not isPseudoSource(peak, config.pseudoColumns) 

201 ] 

202 

203 # Only deblend sources that can be initialized 

204 sources, skipped = init_all_sources( 

205 frame=frame, 

206 centers=centers, 

207 observations=observation, 

208 thresh=config.morphThresh, 

209 max_components=maxComponents, 

210 min_snr=config.minSNR, 

211 shifting=False, 

212 fallback=config.fallback, 

213 silent=config.catchFailures, 

214 set_spectra=spectrumInit, 

215 ) 

216 

217 # Attach the peak to all of the initialized sources 

218 srcIndex = 0 

219 for k, center in enumerate(centers): 

220 if k not in skipped: 

221 # This is just to make sure that there isn't a coding bug 

222 assert np.all(sources[srcIndex].center == center) 

223 # Store the record for the peak with the appropriate source 

224 sources[srcIndex].detectedPeak = footprint.peaks[k] 

225 srcIndex += 1 

226 

227 # Create the blend and attempt to optimize it 

228 blend = Blend(sources, observation) 

229 try: 

230 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError) 

231 except ArithmeticError: 

232 # This occurs when a gradient update produces a NaN value 

233 # This is usually due to a source initialized with a 

234 # negative SED or no flux, often because the peak 

235 # is a noise fluctuation in one band and not a real source. 

236 iterations = len(blend.loss) 

237 failedSources = [] 

238 for k, src in enumerate(sources): 

239 if np.any(~np.isfinite(src.get_model())): 

240 failedSources.append(k) 

241 raise ScarletGradientError(iterations, failedSources) 

242 

243 # Store the location of the PSF center for storage 

244 blend.psfCenter = (psfCenter.x, psfCenter.y) 

245 

246 return blend, skipped 

247 

248 

249def buildLiteObservation( 

250 modelPsf, 

251 psfCenter, 

252 mExposure, 

253 footprint=None, 

254 badPixelMasks=None, 

255 useWeights=True, 

256 convolutionType="real", 

257): 

258 """Generate a LiteObservation from a set of parameters. 

259 

260 Make the generation and reconstruction of a scarlet model consistent 

261 by building a `LiteObservation` from a set of parameters. 

262 

263 Parameters 

264 ---------- 

265 modelPsf : `numpy.ndarray` 

266 The 2D model of the PSF in the partially deconvolved space. 

267 psfCenter : `tuple` or `Point2I` or `Point2D` 

268 The location `(x, y)` used as the center of the PSF. 

269 mExposure : `lsst.afw.image.multiband.MultibandExposure` 

270 The multi-band exposure that the model represents. 

271 If `mExposure` is `None` then no image, variance, or weights are 

272 attached to the observation. 

273 footprint : `lsst.afw.detection.Footprint` 

274 The footprint that is being fit. 

275 If `Footprint` is `None` then the weights are not updated to mask 

276 out pixels not contained in the footprint. 

277 badPixelMasks : `list` of `str` 

278 The keys from the bit mask plane used to mask out pixels 

279 during the fit. 

280 If `badPixelMasks` is `None` then the default values from 

281 `ScarletDeblendConfig.badMask` is used. 

282 useWeights : `bool` 

283 Whether or not fitting should use inverse variance weights to 

284 calculate the log-likelihood. 

285 convolutionType : `str` 

286 The type of convolution to use (either "real" or "fft"). 

287 When reconstructing an image it is advised to use "real" to avoid 

288 polluting the footprint with 

289 

290 Returns 

291 ------- 

292 observation : `scarlet.lite.LiteObservation` 

293 The observation constructed from the input parameters. 

294 """ 

295 # Initialize the observed PSFs 

296 if not isinstance(psfCenter, Point2D): 

297 psfCenter = Point2D(*psfCenter) 

298 psfModels = mExposure.computePsfKernelImage(psfCenter) 

299 

300 # Use the inverse variance as the weights 

301 if useWeights: 

302 weights = 1/mExposure.variance.array 

303 else: 

304 # Mask out bad pixels 

305 weights = np.ones_like(mExposure.image.array) 

306 if badPixelMasks is None: 

307 badPixelMasks = ScarletDeblendConfig().badMask 

308 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks) 

309 mask = mExposure.mask.array & badPixels 

310 weights[mask > 0] = 0 

311 

312 if footprint is not None: 

313 # Mask out the pixels outside the footprint 

314 weights *= footprint.spans.asArray() 

315 

316 return lite.LiteObservation( 

317 images=mExposure.image.array, 

318 variance=mExposure.variance.array, 

319 weights=weights, 

320 psfs=psfModels, 

321 model_psf=modelPsf[None, :, :], 

322 convolution_mode=convolutionType, 

323 ) 

324 

325 

326def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None): 

327 """Deblend a parent footprint 

328 

329 Parameters 

330 ---------- 

331 mExposure : `lsst.image.MultibandExposure` 

332 - The multiband exposure containing the image, 

333 mask, and variance data 

334 footprint : `lsst.detection.Footprint` 

335 - The footprint of the parent to deblend 

336 config : `ScarletDeblendConfig` 

337 - Configuration of the deblending task 

338 """ 

339 # Extract coordinates from each MultiColorPeak 

340 bbox = footprint.getBBox() 

341 psfCenter = footprint.getCentroid() 

342 

343 observation = buildLiteObservation( 

344 modelPsf=modelPsf, 

345 psfCenter=psfCenter, 

346 mExposure=mExposure[:, bbox], 

347 footprint=footprint, 

348 badPixelMasks=config.badMask, 

349 useWeights=config.useWeights, 

350 convolutionType=config.convolutionType, 

351 ) 

352 

353 # Convert the centers to pixel coordinates 

354 xmin = bbox.getMinX() 

355 ymin = bbox.getMinY() 

356 centers = [ 

357 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

358 for peak in footprint.peaks 

359 if not isPseudoSource(peak, config.pseudoColumns) 

360 ] 

361 

362 # Initialize the sources 

363 if config.morphImage == "chi2": 

364 sources = lite.init_all_sources_main( 

365 observation, 

366 centers, 

367 min_snr=config.minSNR, 

368 thresh=config.morphThresh, 

369 ) 

370 elif config.morphImage == "wavelet": 

371 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin()) 

372 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)] 

373 sources = lite.init_all_sources_wavelets( 

374 observation, 

375 centers, 

376 use_psf=False, 

377 wavelets=_wavelets, 

378 min_snr=config.minSNR, 

379 ) 

380 else: 

381 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.") 

382 

383 # Set the optimizer 

384 if config.optimizer == "adaprox": 

385 parameterization = partial( 

386 lite.init_adaprox_component, 

387 bg_thresh=config.backgroundThresh, 

388 max_prox_iter=config.maxProxIter, 

389 ) 

390 elif config.optimizer == "fista": 

391 parameterization = partial( 

392 lite.init_fista_component, 

393 bg_thresh=config.backgroundThresh, 

394 ) 

395 else: 

396 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.") 

397 sources = lite.parameterize_sources(sources, observation, parameterization) 

398 

399 # Attach the peak to all of the initialized sources 

400 for k, center in enumerate(centers): 

401 # This is just to make sure that there isn't a coding bug 

402 if len(sources[k].components) > 0 and np.any(sources[k].center != center): 

403 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}") 

404 # Store the record for the peak with the appropriate source 

405 sources[k].detectedPeak = footprint.peaks[k] 

406 

407 blend = lite.LiteBlend(sources, observation) 

408 

409 # Initialize each source with its best fit spectrum 

410 if spectrumInit: 

411 blend.fit_spectra() 

412 

413 # Set the sources that could not be initialized and were skipped 

414 skipped = [src for src in sources if src.is_null] 

415 

416 blend.fit( 

417 max_iter=config.maxIter, 

418 e_rel=config.relativeError, 

419 min_iter=config.minIter, 

420 reweight=False, 

421 ) 

422 

423 # Store the location of the PSF center for storage 

424 blend.psfCenter = (psfCenter.x, psfCenter.y) 

425 

426 return blend, skipped 

427 

428 

429@dataclass 

430class DeblenderMetrics: 

431 """Metrics and measurements made on single sources. 

432 

433 Store deblender metrics to be added as attributes to a scarlet source 

434 before it is converted into a `SourceRecord`. 

435 When DM-34414 is finished this class will be eliminated and the metrics 

436 will be added to the schema using a pipeline task that calculates them 

437 from the stored deconvolved models. 

438 

439 All of the parameters are one dimensional numpy arrays, 

440 with an element for each band in the observed images. 

441 

442 `maxOverlap` is useful as a metric for determining how blended a source 

443 is because if it only overlaps with other sources at or below 

444 the noise level, it is likely to be a mostly isolated source 

445 in the deconvolved model frame. 

446 

447 `fluxOverlapFraction` is potentially more useful than the canonical 

448 "blendedness" (or purity) metric because it accounts for potential 

449 biases created during deblending by not weighting the overlapping 

450 flux with the flux of this sources model. 

451 

452 Attributes 

453 ---------- 

454 maxOverlap : `numpy.ndarray` 

455 The maximum overlap that the source has with its neighbors in 

456 a single pixel. 

457 fluxOverlap : `numpy.ndarray` 

458 The total flux from neighbors overlapping with the current source. 

459 fluxOverlapFraction : `numpy.ndarray` 

460 The fraction of `flux from neighbors/source flux` for a 

461 given source within the source's footprint. 

462 blendedness : `numpy.ndarray` 

463 The metric for determining how blended a source is using the 

464 Bosch et al. 2018 metric for "blendedness." Note that some 

465 surveys use the term "purity," which is `1-blendedness`. 

466 """ 

467 maxOverlap: np.array 

468 fluxOverlap: np.array 

469 fluxOverlapFraction: np.array 

470 blendedness: np.array 

471 

472 

473def setDeblenderMetrics(blend): 

474 """Set metrics that can be used to evalute the deblender accuracy 

475 

476 This function calculates the `DeblenderMetrics` for each source in the 

477 blend, and assigns it to that sources `metrics` property in place. 

478 

479 Parameters 

480 ---------- 

481 blend : `scarlet.lite.Blend` 

482 The blend containing the sources to measure. 

483 """ 

484 # Store the full model of the scene for comparison 

485 blendModel = blend.get_model() 

486 for k, src in enumerate(blend.sources): 

487 # Extract the source model in the full bounding box 

488 model = src.get_model(bbox=blend.bbox) 

489 # The footprint is the 2D array of non-zero pixels in each band 

490 footprint = np.bitwise_or.reduce(model > 0, axis=0) 

491 # Calculate the metrics. 

492 # See `DeblenderMetrics` for a description of each metric. 

493 neighborOverlap = (blendModel-model) * footprint[None, :, :] 

494 maxOverlap = np.max(neighborOverlap, axis=(1, 2)) 

495 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2)) 

496 fluxModel = np.sum(model, axis=(1, 2)) 

497 fluxOverlapFraction = np.zeros((len(model), ), dtype=float) 

498 isFinite = fluxModel > 0 

499 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite] 

500 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2)) 

501 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness) 

502 

503 

504class ScarletDeblendConfig(pexConfig.Config): 

505 """MultibandDeblendConfig 

506 

507 Configuration for the multiband deblender. 

508 The parameters are organized by the parameter types, which are 

509 - Stopping Criteria: Used to determine if the fit has converged 

510 - Position Fitting Criteria: Used to fit the positions of the peaks 

511 - Constraints: Used to apply constraints to the peaks and their components 

512 - Other: Parameters that don't fit into the above categories 

513 """ 

514 # Stopping Criteria 

515 minIter = pexConfig.Field(dtype=int, default=1, 

516 doc="Minimum number of iterations before the optimizer is allowed to stop.") 

517 maxIter = pexConfig.Field(dtype=int, default=300, 

518 doc=("Maximum number of iterations to deblend a single parent")) 

519 relativeError = pexConfig.Field(dtype=float, default=1e-2, 

520 doc=("Change in the loss function between iterations to exit fitter. " 

521 "Typically this is `1e-2` if measurements will be made on the " 

522 "flux re-distributed models and `1e-4` when making measurements " 

523 "on the models themselves.")) 

524 

525 # Constraints 

526 morphThresh = pexConfig.Field(dtype=float, default=1, 

527 doc="Fraction of background RMS a pixel must have" 

528 "to be included in the initial morphology") 

529 # Lite Parameters 

530 # All of these parameters (except version) are only valid if version='lite' 

531 version = pexConfig.ChoiceField( 

532 dtype=str, 

533 default="lite", 

534 allowed={ 

535 "scarlet": "main scarlet version (likely to be deprecated soon)", 

536 "lite": "Optimized version of scarlet for survey data from a single instrument", 

537 }, 

538 doc="The version of scarlet to use.", 

539 ) 

540 optimizer = pexConfig.ChoiceField( 

541 dtype=str, 

542 default="adaprox", 

543 allowed={ 

544 "adaprox": "Proximal ADAM optimization", 

545 "fista": "Accelerated proximal gradient method", 

546 }, 

547 doc="The optimizer to use for fitting parameters and is only used when version='lite'", 

548 ) 

549 morphImage = pexConfig.ChoiceField( 

550 dtype=str, 

551 default="chi2", 

552 allowed={ 

553 "chi2": "Initialize sources on a chi^2 image made from all available bands", 

554 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image", 

555 }, 

556 doc="The type of image to use for initializing the morphology. " 

557 "Must be either 'chi2' or 'wavelet'. " 

558 ) 

559 backgroundThresh = pexConfig.Field( 

560 dtype=float, 

561 default=0.25, 

562 doc="Fraction of background to use for a sparsity threshold. " 

563 "This prevents sources from growing unrealistically outside " 

564 "the parent footprint while still modeling flux correctly " 

565 "for bright sources." 

566 ) 

567 maxProxIter = pexConfig.Field( 

568 dtype=int, 

569 default=1, 

570 doc="Maximum number of proximal operator iterations inside of each " 

571 "iteration of the optimizer. " 

572 "This config field is only used if version='lite' and optimizer='adaprox'." 

573 ) 

574 waveletScales = pexConfig.Field( 

575 dtype=int, 

576 default=5, 

577 doc="Number of wavelet scales to use for wavelet initialization. " 

578 "This field is only used when `version`='lite' and `morphImage`='wavelet'." 

579 ) 

580 

581 # Other scarlet paremeters 

582 useWeights = pexConfig.Field( 

583 dtype=bool, default=True, 

584 doc=("Whether or not use use inverse variance weighting." 

585 "If `useWeights` is `False` then flat weights are used")) 

586 modelPsfSize = pexConfig.Field( 

587 dtype=int, default=11, 

588 doc="Model PSF side length in pixels") 

589 modelPsfSigma = pexConfig.Field( 

590 dtype=float, default=0.8, 

591 doc="Define sigma for the model frame PSF") 

592 minSNR = pexConfig.Field( 

593 dtype=float, default=50, 

594 doc="Minimum Signal to noise to accept the source." 

595 "Sources with lower flux will be initialized with the PSF but updated " 

596 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).") 

597 saveTemplates = pexConfig.Field( 

598 dtype=bool, default=True, 

599 doc="Whether or not to save the SEDs and templates") 

600 processSingles = pexConfig.Field( 

601 dtype=bool, default=True, 

602 doc="Whether or not to process isolated sources in the deblender") 

603 convolutionType = pexConfig.Field( 

604 dtype=str, default="fft", 

605 doc="Type of convolution to render the model to the observations.\n" 

606 "- 'fft': perform convolutions in Fourier space\n" 

607 "- 'real': peform convolutions in real space.") 

608 sourceModel = pexConfig.Field( 

609 dtype=str, default="double", 

610 doc=("How to determine which model to use for sources, from\n" 

611 "- 'single': use a single component for all sources\n" 

612 "- 'double': use a bulge disk model for all sources\n" 

613 "- 'compact': use a single component model, initialzed with a point source morphology, " 

614 " for all sources\n" 

615 "- 'point': use a point-source model for all sources\n" 

616 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"), 

617 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.", 

618 ) 

619 setSpectra = pexConfig.Field( 

620 dtype=bool, default=True, 

621 doc="Whether or not to solve for the best-fit spectra during initialization. " 

622 "This makes initialization slightly longer, as it requires a convolution " 

623 "to set the optimal spectra, but results in a much better initial log-likelihood " 

624 "and reduced total runtime, with convergence in fewer iterations." 

625 "This option is only used when " 

626 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.") 

627 

628 # Mask-plane restrictions 

629 badMask = pexConfig.ListField( 

630 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"], 

631 doc="Whether or not to process isolated sources in the deblender") 

632 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

633 doc="Mask planes to ignore when performing statistics") 

634 maskLimits = pexConfig.DictField( 

635 keytype=str, 

636 itemtype=float, 

637 default={}, 

638 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

639 "Sources violating this limit will not be deblended. " 

640 "If the fraction is `0` then the limit is a single pixel."), 

641 ) 

642 

643 # Size restrictions 

644 maxNumberOfPeaks = pexConfig.Field( 

645 dtype=int, default=200, 

646 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

647 " (<= 0: unlimited)")) 

648 maxFootprintArea = pexConfig.Field( 

649 dtype=int, default=100_000, 

650 doc=("Maximum area for footprints before they are ignored as large; " 

651 "non-positive means no threshold applied")) 

652 maxAreaTimesPeaks = pexConfig.Field( 

653 dtype=int, default=10_000_000, 

654 doc=("Maximum rectangular footprint area * nPeaks in the footprint. " 

655 "This was introduced in DM-33690 to prevent fields that are crowded or have a " 

656 "LSB galaxy that causes memory intensive initialization in scarlet from dominating " 

657 "the overall runtime and/or causing the task to run out of memory. " 

658 "(<= 0: unlimited)") 

659 ) 

660 maxFootprintSize = pexConfig.Field( 

661 dtype=int, default=0, 

662 doc=("Maximum linear dimension for footprints before they are ignored " 

663 "as large; non-positive means no threshold applied")) 

664 minFootprintAxisRatio = pexConfig.Field( 

665 dtype=float, default=0.0, 

666 doc=("Minimum axis ratio for footprints before they are ignored " 

667 "as large; non-positive means no threshold applied")) 

668 maxSpectrumCutoff = pexConfig.Field( 

669 dtype=int, default=1_000_000, 

670 doc=("Maximum number of pixels * number of sources in a blend. " 

671 "This is different than `maxFootprintArea` because this isn't " 

672 "the footprint area but the area of the bounding box that " 

673 "contains the footprint, and is also multiplied by the number of" 

674 "sources in the footprint. This prevents large skinny blends with " 

675 "a high density of sources from running out of memory. " 

676 "If `maxSpectrumCutoff == -1` then there is no cutoff.") 

677 ) 

678 # Failure modes 

679 fallback = pexConfig.Field( 

680 dtype=bool, default=True, 

681 doc="Whether or not to fallback to a smaller number of components if a source does not initialize" 

682 ) 

683 notDeblendedMask = pexConfig.Field( 

684 dtype=str, default="NOT_DEBLENDED", optional=True, 

685 doc="Mask name for footprints not deblended, or None") 

686 catchFailures = pexConfig.Field( 

687 dtype=bool, default=True, 

688 doc=("If True, catch exceptions thrown by the deblender, log them, " 

689 "and set a flag on the parent, instead of letting them propagate up")) 

690 

691 # Other options 

692 columnInheritance = pexConfig.DictField( 

693 keytype=str, itemtype=str, default={ 

694 "deblend_nChild": "deblend_parentNChild", 

695 "deblend_nPeaks": "deblend_parentNPeaks", 

696 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag", 

697 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag", 

698 }, 

699 doc="Columns to pass from the parent to the child. " 

700 "The key is the name of the column for the parent record, " 

701 "the value is the name of the column to use for the child." 

702 ) 

703 pseudoColumns = pexConfig.ListField( 

704 dtype=str, default=['merge_peak_sky', 'sky_source'], 

705 doc="Names of flags which should never be deblended." 

706 ) 

707 

708 # Testing options 

709 # Some obs packages and ci packages run the full pipeline on a small 

710 # subset of data to test that the pipeline is functioning properly. 

711 # This is not meant as scientific validation, so it can be useful 

712 # to only run on a small subset of the data that is large enough to 

713 # test the desired pipeline features but not so long that the deblender 

714 # is the tall pole in terms of execution times. 

715 useCiLimits = pexConfig.Field( 

716 dtype=bool, default=False, 

717 doc="Limit the number of sources deblended for CI to prevent long build times") 

718 ciDeblendChildRange = pexConfig.ListField( 

719 dtype=int, default=[5, 10], 

720 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated." 

721 "If `useCiLimits==False` then this parameter is ignored.") 

722 ciNumParentsToDeblend = pexConfig.Field( 

723 dtype=int, default=10, 

724 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count " 

725 "within `ciDebledChildRange`. " 

726 "If `useCiLimits==False` then this parameter is ignored.") 

727 

728 

729class ScarletDeblendTask(pipeBase.Task): 

730 """ScarletDeblendTask 

731 

732 Split blended sources into individual sources. 

733 

734 This task has no return value; it only modifies the SourceCatalog in-place. 

735 """ 

736 ConfigClass = ScarletDeblendConfig 

737 _DefaultName = "scarletDeblend" 

738 

739 def __init__(self, schema, peakSchema=None, **kwargs): 

740 """Create the task, adding necessary fields to the given schema. 

741 

742 Parameters 

743 ---------- 

744 schema : `lsst.afw.table.schema.schema.Schema` 

745 Schema object for measurement fields; will be modified in-place. 

746 peakSchema : `lsst.afw.table.schema.schema.Schema` 

747 Schema of Footprint Peaks that will be passed to the deblender. 

748 Any fields beyond the PeakTable minimal schema will be transferred 

749 to the main source Schema. If None, no fields will be transferred 

750 from the Peaks. 

751 filters : list of str 

752 Names of the filters used for the eposures. This is needed to store 

753 the SED as a field 

754 **kwargs 

755 Passed to Task.__init__. 

756 """ 

757 pipeBase.Task.__init__(self, **kwargs) 

758 

759 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

760 if peakSchema is None: 

761 # In this case, the peakSchemaMapper will transfer nothing, but 

762 # we'll still have one 

763 # to simplify downstream code 

764 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

765 else: 

766 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

767 for item in peakSchema: 

768 if item.key not in peakMinimalSchema: 

769 self.peakSchemaMapper.addMapping(item.key, item.field) 

770 # Because SchemaMapper makes a copy of the output schema 

771 # you give its ctor, it isn't updating this Schema in 

772 # place. That's probably a design flaw, but in the 

773 # meantime, we'll keep that schema in sync with the 

774 # peakSchemaMapper.getOutputSchema() manually, by adding 

775 # the same fields to both. 

776 schema.addField(item.field) 

777 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

778 self._addSchemaKeys(schema) 

779 self.schema = schema 

780 self.toCopyFromParent = [item.key for item in self.schema 

781 if item.field.getName().startswith("merge_footprint")] 

782 

783 def _addSchemaKeys(self, schema): 

784 """Add deblender specific keys to the schema 

785 """ 

786 # Parent (blend) fields 

787 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms') 

788 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge') 

789 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

790 doc='Number of children this object has (defaults to 0)') 

791 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32, 

792 doc="Number of initial peaks in the blend. " 

793 "This includes peaks that may have been culled " 

794 "during deblending or failed to deblend") 

795 # Skipped flags 

796 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

797 doc="Deblender skipped this source") 

798 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag', 

799 doc='The source has only a single peak ' 

800 'and was not deblended') 

801 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag', 

802 doc='The source is identified as a "pseudo" source and ' 

803 'was not deblended') 

804 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

805 doc='Source had too many peaks; ' 

806 'only the brightest were included') 

807 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

808 doc='Parent footprint covered too many pixels') 

809 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

810 doc='Parent footprint had too many masked pixels') 

811 # Convergence flags 

812 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag', 

813 doc='scarlet sed optimization did not converge before' 

814 'config.maxIter') 

815 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag', 

816 doc='scarlet morph optimization did not converge before' 

817 'config.maxIter') 

818 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag', 

819 type='Flag', 

820 doc='at least one source in the blend' 

821 'failed to converge') 

822 # Error flags 

823 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

824 doc="Deblending failed on source") 

825 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25, 

826 doc='Name of error if the blend failed') 

827 # Deblended source fields 

828 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center", 

829 doc="Center used to apply constraints in scarlet", 

830 unit="pixel") 

831 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32, 

832 doc="ID of the peak in the parent footprint. " 

833 "This is not unique, but the combination of 'parent'" 

834 "and 'peakId' should be for all child sources. " 

835 "Top level blends with no parents have 'peakId=0'") 

836 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count', 

837 doc="The instFlux at the peak position of deblended mode") 

838 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25, 

839 doc="The type of model used, for example " 

840 "MultiExtendedSource, SingleExtendedSource, PointSource") 

841 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32, 

842 doc="deblend_nPeaks from this records parent.") 

843 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32, 

844 doc="deblend_nChild from this records parent.") 

845 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32, 

846 doc="Flux measurement from scarlet") 

847 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32, 

848 doc="Final logL, used to identify regressions in scarlet.") 

849 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag', 

850 doc='Source had flux on the edge of the parent footprint') 

851 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag', 

852 doc="True when scarlet initializes sources " 

853 "in the blend with a more accurate spectrum. " 

854 "The algorithm uses a lot of memory, " 

855 "so large dense blends will use " 

856 "a less accurate initialization.") 

857 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32, 

858 doc="Number of components in a ScarletLiteSource. " 

859 "If `config.version != 'lite'`then " 

860 "this column is set to zero.") 

861 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

862 doc='Deblender thought this source looked like a PSF') 

863 # Blendedness/classification metrics 

864 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32, 

865 doc="Maximum overlap with all of the other neighbors flux " 

866 "combined." 

867 "This is useful as a metric for determining how blended a " 

868 "source is because if it only overlaps with other sources " 

869 "at or below the noise level, it is likely to be a mostly " 

870 "isolated source in the deconvolved model frame.") 

871 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32, 

872 doc="This is the total flux from neighboring objects that " 

873 "overlaps with this source.") 

874 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32, 

875 doc="This is the fraction of " 

876 "`flux from neighbors/source flux` " 

877 "for a given source within the source's" 

878 "footprint.") 

879 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32, 

880 doc="The Bosch et al. 2018 metric for 'blendedness.' ") 

881 

882 @timeMethod 

883 def run(self, mExposure, mergedSources): 

884 """Get the psf from each exposure and then run deblend(). 

885 

886 Parameters 

887 ---------- 

888 mExposure : `MultibandExposure` 

889 The exposures should be co-added images of the same 

890 shape and region of the sky. 

891 mergedSources : `SourceCatalog` 

892 The merged `SourceCatalog` that contains parent footprints 

893 to (potentially) deblend. 

894 

895 Returns 

896 ------- 

897 templateCatalogs: dict 

898 Keys are the names of the filters and the values are 

899 `lsst.afw.table.source.source.SourceCatalog`'s. 

900 These are catalogs with heavy footprints that are the templates 

901 created by the multiband templates. 

902 """ 

903 return self.deblend(mExposure, mergedSources) 

904 

905 @timeMethod 

906 def deblend(self, mExposure, catalog): 

907 """Deblend a data cube of multiband images 

908 

909 Parameters 

910 ---------- 

911 mExposure : `MultibandExposure` 

912 The exposures should be co-added images of the same 

913 shape and region of the sky. 

914 catalog : `SourceCatalog` 

915 The merged `SourceCatalog` that contains parent footprints 

916 to (potentially) deblend. The new deblended sources are 

917 appended to this catalog in place. 

918 

919 Returns 

920 ------- 

921 catalogs : `dict` or `None` 

922 Keys are the names of the filters and the values are 

923 `lsst.afw.table.source.source.SourceCatalog`'s. 

924 These are catalogs with heavy footprints that are the templates 

925 created by the multiband templates. 

926 """ 

927 import time 

928 

929 # Cull footprints if required by ci 

930 if self.config.useCiLimits: 

931 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.", 

932 len(catalog)) 

933 # Select parents with a number of children in the range 

934 # config.ciDeblendChildRange 

935 minChildren, maxChildren = self.config.ciDeblendChildRange 

936 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog]) 

937 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0] 

938 if len(childrenInRange) < self.config.ciNumParentsToDeblend: 

939 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range " 

940 "indicated by ciDeblendChildRange. Adjust this range to include more " 

941 "parents.") 

942 # Keep all of the isolated parents and the first 

943 # `ciNumParentsToDeblend` children 

944 parents = nPeaks == 1 

945 children = np.zeros((len(catalog),), dtype=bool) 

946 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True 

947 catalog = catalog[parents | children] 

948 # We need to update the IdFactory, otherwise the the source ids 

949 # will not be sequential 

950 idFactory = catalog.getIdFactory() 

951 maxId = np.max(catalog["id"]) 

952 idFactory.notify(maxId) 

953 

954 filters = mExposure.filters 

955 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure)) 

956 periodicLog = PeriodicLogger(self.log) 

957 

958 # Create a set of wavelet coefficients if using wavelet initialization 

959 if self.config.version == "lite" and self.config.morphImage == "wavelet": 

960 images = mExposure.image.array 

961 variance = mExposure.variance.array 

962 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales) 

963 else: 

964 wavelets = None 

965 

966 # Add the NOT_DEBLENDED mask to the mask plane in each band 

967 if self.config.notDeblendedMask: 

968 for mask in mExposure.mask: 

969 mask.addMaskPlane(self.config.notDeblendedMask) 

970 

971 # Initialize the persistable data model 

972 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma) 

973 dataModel = ScarletModelData(filters, modelPsf) 

974 

975 nParents = len(catalog) 

976 nDeblendedParents = 0 

977 skippedParents = [] 

978 for parentIndex in range(nParents): 

979 parent = catalog[parentIndex] 

980 foot = parent.getFootprint() 

981 bbox = foot.getBBox() 

982 peaks = foot.getPeaks() 

983 

984 # Since we use the first peak for the parent object, we should 

985 # propagate its flags to the parent source. 

986 parent.assign(peaks[0], self.peakSchemaMapper) 

987 

988 # Block of conditions for skipping a parent with multiple children 

989 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None: 

990 self._skipParent(parent, *skipArgs) 

991 skippedParents.append(parentIndex) 

992 continue 

993 

994 nDeblendedParents += 1 

995 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks)) 

996 # Run the deblender 

997 blendError = None 

998 

999 # Choose whether or not to use improved spectral initialization. 

1000 # This significantly cuts down on the number of iterations 

1001 # that the optimizer needs and usually results in a better 

1002 # fit. 

1003 # But using least squares on a very large blend causes memory 

1004 # issues, so it is not done for large blends 

1005 if self.config.setSpectra: 

1006 if self.config.maxSpectrumCutoff <= 0: 

1007 spectrumInit = True 

1008 else: 

1009 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff 

1010 else: 

1011 spectrumInit = False 

1012 

1013 try: 

1014 t0 = time.monotonic() 

1015 # Build the parameter lists with the same ordering 

1016 if self.config.version == "scarlet": 

1017 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit) 

1018 elif self.config.version == "lite": 

1019 blend, skipped = deblend_lite( 

1020 mExposure=mExposure, 

1021 modelPsf=modelPsf, 

1022 footprint=foot, 

1023 config=self.config, 

1024 spectrumInit=spectrumInit, 

1025 wavelets=wavelets, 

1026 ) 

1027 tf = time.monotonic() 

1028 runtime = (tf-t0)*1000 

1029 converged = _checkBlendConvergence(blend, self.config.relativeError) 

1030 # Store the number of components in the blend 

1031 if self.config.version == "lite": 

1032 nComponents = len(blend.components) 

1033 else: 

1034 nComponents = 0 

1035 nChild = len(blend.sources) 

1036 # Catch all errors and filter out the ones that we know about 

1037 except Exception as e: 

1038 print("deblend failed") 

1039 print(e) 

1040 blendError = type(e).__name__ 

1041 if isinstance(e, ScarletGradientError): 

1042 parent.set(self.iterKey, e.iterations) 

1043 elif not isinstance(e, IncompleteDataError): 

1044 blendError = "UnknownError" 

1045 if self.config.catchFailures: 

1046 # Make it easy to find UnknownErrors in the log file 

1047 self.log.warn("UnknownError") 

1048 import traceback 

1049 traceback.print_exc() 

1050 else: 

1051 raise 

1052 

1053 self._skipParent( 

1054 parent=parent, 

1055 skipKey=self.deblendFailedKey, 

1056 logMessage=f"Unable to deblend source {parent.getId}: {blendError}", 

1057 ) 

1058 parent.set(self.deblendErrorKey, blendError) 

1059 skippedParents.append(parentIndex) 

1060 continue 

1061 

1062 # Update the parent record with the deblending results 

1063 if self.config.version == "scarlet": 

1064 logL = -blend.loss[-1] + blend.observations[0].log_norm 

1065 elif self.config.version == "lite": 

1066 logL = blend.loss[-1] 

1067 self._updateParentRecord( 

1068 parent=parent, 

1069 nPeaks=len(peaks), 

1070 nChild=nChild, 

1071 nComponents=nComponents, 

1072 runtime=runtime, 

1073 iterations=len(blend.loss), 

1074 logL=logL, 

1075 spectrumInit=spectrumInit, 

1076 converged=converged, 

1077 ) 

1078 

1079 # Add each deblended source to the catalog 

1080 for k, scarletSource in enumerate(blend.sources): 

1081 # Skip any sources with no flux or that scarlet skipped because 

1082 # it could not initialize 

1083 if k in skipped or (self.config.version == "lite" and scarletSource.is_null): 

1084 # No need to propagate anything 

1085 continue 

1086 parent.set(self.deblendSkippedKey, False) 

1087 

1088 # Add all fields except the HeavyFootprint to the 

1089 # source record 

1090 sourceRecord = self._addChild( 

1091 parent=parent, 

1092 peak=scarletSource.detectedPeak, 

1093 catalog=catalog, 

1094 scarletSource=scarletSource, 

1095 ) 

1096 scarletSource.recordId = sourceRecord.getId() 

1097 scarletSource.peakId = scarletSource.detectedPeak.getId() 

1098 

1099 # Store the blend information so that it can be persisted 

1100 if self.config.version == "lite": 

1101 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin()) 

1102 else: 

1103 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin()) 

1104 dataModel.blends[parent.getId()] = blendData 

1105 

1106 # Log a message if it has been a while since the last log. 

1107 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents) 

1108 

1109 # Clear the cached values in scarlet to clear out memory 

1110 scarlet.cache.Cache._cache = {} 

1111 

1112 # Update the mExposure mask with the footprint of skipped parents 

1113 if self.config.notDeblendedMask: 

1114 for mask in mExposure.mask: 

1115 for parentIndex in skippedParents: 

1116 fp = catalog[parentIndex].getFootprint() 

1117 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask)) 

1118 

1119 self.log.info("Deblender results: of %d parent sources, %d were deblended, " 

1120 "creating %d children, for a total of %d sources", 

1121 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog)) 

1122 return catalog, dataModel 

1123 

1124 def _isLargeFootprint(self, footprint): 

1125 """Returns whether a Footprint is large 

1126 

1127 'Large' is defined by thresholds on the area, size and axis ratio, 

1128 and total area of the bounding box multiplied by 

1129 the number of children. 

1130 These may be disabled independently by configuring them to be 

1131 non-positive. 

1132 """ 

1133 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

1134 return True 

1135 if self.config.maxFootprintSize > 0: 

1136 bbox = footprint.getBBox() 

1137 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

1138 return True 

1139 if self.config.minFootprintAxisRatio > 0: 

1140 axes = afwEll.Axes(footprint.getShape()) 

1141 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

1142 return True 

1143 if self.config.maxAreaTimesPeaks > 0: 

1144 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks: 

1145 return True 

1146 return False 

1147 

1148 def _isMasked(self, footprint, mExposure): 

1149 """Returns whether the footprint violates the mask limits 

1150 

1151 Parameters 

1152 ---------- 

1153 footprint : `lsst.afw.detection.Footprint` 

1154 The footprint to check for masked pixels 

1155 mMask : `lsst.afw.image.MaskX` 

1156 The mask plane to check for masked pixels in the `footprint`. 

1157 

1158 Returns 

1159 ------- 

1160 isMasked : `bool` 

1161 `True` if `self.config.maskPlaneLimits` is less than the 

1162 fraction of pixels for a given mask in 

1163 `self.config.maskLimits`. 

1164 """ 

1165 bbox = footprint.getBBox() 

1166 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0) 

1167 size = float(footprint.getArea()) 

1168 for maskName, limit in self.config.maskLimits.items(): 

1169 maskVal = mExposure.mask.getPlaneBitMask(maskName) 

1170 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin()) 

1171 # spanset of masked pixels 

1172 maskedSpan = footprint.spans.intersect(_mask, maskVal) 

1173 if (maskedSpan.getArea())/size > limit: 

1174 return True 

1175 return False 

1176 

1177 def _skipParent(self, parent, skipKey, logMessage): 

1178 """Update a parent record that is not being deblended. 

1179 

1180 This is a fairly trivial function but is implemented to ensure 

1181 that a skipped parent updates the appropriate columns 

1182 consistently, and always has a flag to mark the reason that 

1183 it is being skipped. 

1184 

1185 Parameters 

1186 ---------- 

1187 parent : `lsst.afw.table.source.source.SourceRecord` 

1188 The parent record to flag as skipped. 

1189 skipKey : `bool` 

1190 The name of the flag to mark the reason for skipping. 

1191 logMessage : `str` 

1192 The message to display in a log.trace when a source 

1193 is skipped. 

1194 """ 

1195 if logMessage is not None: 

1196 self.log.trace(logMessage) 

1197 self._updateParentRecord( 

1198 parent=parent, 

1199 nPeaks=len(parent.getFootprint().peaks), 

1200 nChild=0, 

1201 nComponents=0, 

1202 runtime=np.nan, 

1203 iterations=0, 

1204 logL=np.nan, 

1205 spectrumInit=False, 

1206 converged=False, 

1207 ) 

1208 

1209 # Mark the source as skipped by the deblender and 

1210 # flag the reason why. 

1211 parent.set(self.deblendSkippedKey, True) 

1212 parent.set(skipKey, True) 

1213 

1214 def _checkSkipped(self, parent, mExposure): 

1215 """Update a parent record that is not being deblended. 

1216 

1217 This is a fairly trivial function but is implemented to ensure 

1218 that a skipped parent updates the appropriate columns 

1219 consistently, and always has a flag to mark the reason that 

1220 it is being skipped. 

1221 

1222 Parameters 

1223 ---------- 

1224 parent : `lsst.afw.table.source.source.SourceRecord` 

1225 The parent record to flag as skipped. 

1226 mExposure : `MultibandExposure` 

1227 The exposures should be co-added images of the same 

1228 shape and region of the sky. 

1229 Returns 

1230 ------- 

1231 skip: `bool` 

1232 `True` if the deblender will skip the parent 

1233 """ 

1234 skipKey = None 

1235 skipMessage = None 

1236 footprint = parent.getFootprint() 

1237 if len(footprint.peaks) < 2 and not self.config.processSingles: 

1238 # Skip isolated sources unless processSingles is turned on. 

1239 # Note: this does not flag isolated sources as skipped or 

1240 # set the NOT_DEBLENDED mask in the exposure, 

1241 # since these aren't really any skipped blends. 

1242 skipKey = self.isolatedParentKey 

1243 elif isPseudoSource(parent, self.config.pseudoColumns): 

1244 # We also skip pseudo sources, like sky objects, which 

1245 # are intended to be skipped. 

1246 skipKey = self.pseudoKey 

1247 if self._isLargeFootprint(footprint): 

1248 # The footprint is above the maximum footprint size limit 

1249 skipKey = self.tooBigKey 

1250 skipMessage = f"Parent {parent.getId()}: skipping large footprint" 

1251 elif self._isMasked(footprint, mExposure): 

1252 # The footprint exceeds the maximum number of masked pixels 

1253 skipKey = self.maskedKey 

1254 skipMessage = f"Parent {parent.getId()}: skipping masked footprint" 

1255 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks: 

1256 # Unlike meas_deblender, in scarlet we skip the entire blend 

1257 # if the number of peaks exceeds max peaks, since neglecting 

1258 # to model any peaks often results in catastrophic failure 

1259 # of scarlet to generate models for the brighter sources. 

1260 skipKey = self.tooManyPeaksKey 

1261 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks" 

1262 if skipKey is not None: 

1263 return (skipKey, skipMessage) 

1264 return None 

1265 

1266 def setSkipFlags(self, mExposure, catalog): 

1267 """Set the skip flags for all of the parent sources 

1268 

1269 This is mostly used for testing which parent sources will be deblended 

1270 and which will be skipped based on the current configuration options. 

1271 Skipped sources will have the appropriate flags set in place in the 

1272 catalog. 

1273 

1274 Parameters 

1275 ---------- 

1276 mExposure : `MultibandExposure` 

1277 The exposures should be co-added images of the same 

1278 shape and region of the sky. 

1279 catalog : `SourceCatalog` 

1280 The merged `SourceCatalog` that contains parent footprints 

1281 to (potentially) deblend. The new deblended sources are 

1282 appended to this catalog in place. 

1283 """ 

1284 for src in catalog: 

1285 if skipArgs := self._checkSkipped(src, mExposure) is not None: 

1286 self._skipParent(src, *skipArgs) 

1287 

1288 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents, 

1289 runtime, iterations, logL, spectrumInit, converged): 

1290 """Update a parent record in all of the single band catalogs. 

1291 

1292 Ensure that all locations that update a parent record, 

1293 whether it is skipped or updated after deblending, 

1294 update all of the appropriate columns. 

1295 

1296 Parameters 

1297 ---------- 

1298 parent : `lsst.afw.table.source.source.SourceRecord` 

1299 The parent record to update. 

1300 nPeaks : `int` 

1301 Number of peaks in the parent footprint. 

1302 nChild : `int` 

1303 Number of children deblended from the parent. 

1304 This may differ from `nPeaks` if some of the peaks 

1305 were culled and have no deblended model. 

1306 nComponents : `int` 

1307 Total number of components in the parent. 

1308 This is usually different than the number of children, 

1309 since it is common for a single source to have multiple 

1310 components. 

1311 runtime : `float` 

1312 Total runtime for deblending. 

1313 iterations : `int` 

1314 Total number of iterations in scarlet before convergence. 

1315 logL : `float` 

1316 Final log likelihood of the blend. 

1317 spectrumInit : `bool` 

1318 True when scarlet used `set_spectra` to initialize all 

1319 sources with better initial intensities. 

1320 converged : `bool` 

1321 True when the optimizer reached convergence before 

1322 reaching the maximum number of iterations. 

1323 """ 

1324 parent.set(self.nPeaksKey, nPeaks) 

1325 parent.set(self.nChildKey, nChild) 

1326 parent.set(self.nComponentsKey, nComponents) 

1327 parent.set(self.runtimeKey, runtime) 

1328 parent.set(self.iterKey, iterations) 

1329 parent.set(self.scarletLogLKey, logL) 

1330 parent.set(self.scarletSpectrumInitKey, spectrumInit) 

1331 parent.set(self.blendConvergenceFailedFlagKey, converged) 

1332 

1333 def _addChild(self, parent, peak, catalog, scarletSource): 

1334 """Add a child to a catalog. 

1335 

1336 This creates a new child in the source catalog, 

1337 assigning it a parent id, and adding all columns 

1338 that are independent across all filter bands. 

1339 

1340 Parameters 

1341 ---------- 

1342 parent : `lsst.afw.table.source.source.SourceRecord` 

1343 The parent of the new child record. 

1344 peak : `lsst.afw.table.PeakRecord` 

1345 The peak record for the peak from the parent peak catalog. 

1346 catalog : `lsst.afw.table.source.source.SourceCatalog` 

1347 The merged `SourceCatalog` that contains parent footprints 

1348 to (potentially) deblend. 

1349 scarletSource : `scarlet.Component` 

1350 The scarlet model for the new source record. 

1351 """ 

1352 src = catalog.addNew() 

1353 for key in self.toCopyFromParent: 

1354 src.set(key, parent.get(key)) 

1355 # The peak catalog is the same for all bands, 

1356 # so we just use the first peak catalog 

1357 src.assign(peak, self.peakSchemaMapper) 

1358 src.setParent(parent.getId()) 

1359 src.set(self.nPeaksKey, 1) 

1360 # Set the psf key based on whether or not the source was 

1361 # deblended using the PointSource model. 

1362 # This key is not that useful anymore since we now keep track of 

1363 # `modelType`, but we continue to propagate it in case code downstream 

1364 # is expecting it. 

1365 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource") 

1366 src.set(self.modelTypeKey, scarletSource.__class__.__name__) 

1367 # We set the runtime to zero so that summing up the 

1368 # runtime column will give the total time spent 

1369 # running the deblender for the catalog. 

1370 src.set(self.runtimeKey, 0) 

1371 

1372 # Set the position of the peak from the parent footprint 

1373 # This will make it easier to match the same source across 

1374 # deblenders and across observations, where the peak 

1375 # position is unlikely to change unless enough time passes 

1376 # for a source to move on the sky. 

1377 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"])) 

1378 src.set(self.peakIdKey, peak["id"]) 

1379 

1380 # Store the number of components for the source 

1381 src.set(self.nComponentsKey, len(scarletSource.components)) 

1382 

1383 # Propagate columns from the parent to the child 

1384 for parentColumn, childColumn in self.config.columnInheritance.items(): 

1385 src.set(childColumn, parent.get(parentColumn)) 

1386 

1387 return src