Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 16%

464 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-14 04:27 -0700

1# This file is part of meas_extensions_scarlet. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from dataclasses import dataclass 

23from functools import partial 

24import logging 

25import numpy as np 

26import scarlet 

27from scarlet.psf import ImagePSF, GaussianPSF 

28from scarlet import Blend, Frame, Observation 

29from scarlet.renderer import ConvolutionRenderer 

30from scarlet.detect import get_detect_wavelets 

31from scarlet.initialization import init_all_sources 

32from scarlet import lite 

33 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.geom as geom 

37import lsst.afw.geom.ellipses as afwEll 

38import lsst.afw.image as afwImage 

39import lsst.afw.detection as afwDet 

40import lsst.afw.table as afwTable 

41from lsst.utils.logging import PeriodicLogger 

42from lsst.utils.timer import timeMethod 

43 

44from .source import bboxToScarletBox 

45from .io import ScarletModelData, scarletToData, scarletLiteToData 

46 

47# Scarlet and proxmin have a different definition of log levels than the stack, 

48# so even "warnings" occur far more often than we would like. 

49# So for now we only display scarlet and proxmin errors, as all other 

50# scarlet outputs would be considered "TRACE" by our standards. 

51scarletLogger = logging.getLogger("scarlet") 

52scarletLogger.setLevel(logging.ERROR) 

53proxminLogger = logging.getLogger("proxmin") 

54proxminLogger.setLevel(logging.ERROR) 

55 

56__all__ = ["deblend", "deblend_lite", "ScarletDeblendConfig", "ScarletDeblendTask"] 

57 

58logger = logging.getLogger(__name__) 

59 

60 

61class IncompleteDataError(Exception): 

62 """The PSF could not be computed due to incomplete data 

63 """ 

64 pass 

65 

66 

67class ScarletGradientError(Exception): 

68 """An error occurred during optimization 

69 

70 This error occurs when the optimizer encounters 

71 a NaN value while calculating the gradient. 

72 """ 

73 def __init__(self, iterations, sources): 

74 self.iterations = iterations 

75 self.sources = sources 

76 msg = ("ScalarGradientError in iteration {0}. " 

77 "NaN values introduced in sources {1}") 

78 self.message = msg.format(iterations, sources) 

79 

80 def __str__(self): 

81 return self.message 

82 

83 

84def _checkBlendConvergence(blend, f_rel): 

85 """Check whether or not a blend has converged 

86 """ 

87 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1]) 

88 convergence = f_rel * np.abs(blend.loss[-1]) 

89 return deltaLoss < convergence 

90 

91 

92def isPseudoSource(source, pseudoColumns): 

93 """Check if a source is a pseudo source. 

94 

95 This is mostly for skipping sky objects, 

96 but any other column can also be added to disable 

97 deblending on a parent or individual source when 

98 set to `True`. 

99 

100 Parameters 

101 ---------- 

102 source : `lsst.afw.table.source.source.SourceRecord` 

103 The source to check for the pseudo bit. 

104 pseudoColumns : `list` of `str` 

105 A list of columns to check for pseudo sources. 

106 """ 

107 isPseudo = False 

108 for col in pseudoColumns: 

109 try: 

110 isPseudo |= source[col] 

111 except KeyError: 

112 pass 

113 return isPseudo 

114 

115 

116def computePsfKernelImage(mExposure, psfCenter): 

117 """Compute the PSF kernel image and update the multiband exposure 

118 if not all of the PSF images could be computed. 

119 

120 Parameters 

121 ---------- 

122 psfCenter : `tuple` or `Point2I` or `Point2D` 

123 The location `(x, y)` used as the center of the PSF. 

124 

125 Returns 

126 ------- 

127 psfModels : `np.ndarray` 

128 The multiband PSF image 

129 mExposure : `MultibandExposure` 

130 The exposure, updated to only use bands that 

131 successfully generated a PSF image. 

132 """ 

133 if not isinstance(psfCenter, geom.Point2D): 

134 psfCenter = geom.Point2D(*psfCenter) 

135 

136 try: 

137 psfModels = mExposure.computePsfKernelImage(psfCenter) 

138 except IncompleteDataError as e: 

139 psfModels = e.partialPsf 

140 # Use only the bands that successfully generated a PSF image. 

141 bands = psfModels.filters 

142 mExposure = mExposure[bands, ] 

143 if len(bands) == 1: 

144 # Only a single band generated a PSF, so the MultibandExposure 

145 # became a single band ExposureF. 

146 # Convert the result back into a MultibandExposure. 

147 mExposure = afwImage.MultibandExposure.fromExposures(bands, [mExposure]) 

148 return psfModels.array, mExposure 

149 

150 

151def deblend(mExposure, footprint, config, spectrumInit): 

152 """Deblend a parent footprint 

153 

154 Parameters 

155 ---------- 

156 mExposure : `lsst.image.MultibandExposure` 

157 The multiband exposure containing the image, 

158 mask, and variance data. 

159 footprint : `lsst.detection.Footprint` 

160 The footprint of the parent to deblend. 

161 config : `ScarletDeblendConfig` 

162 Configuration of the deblending task. 

163 spectrumInit : `bool` 

164 Whether or not to initialize the spectrum. 

165 

166 Returns 

167 ------- 

168 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData` 

169 The persistable representation of a `scarlet.Blend`. 

170 skipped : `list` of `int` 

171 The indices of any children that failed to initialize 

172 and were skipped. 

173 """ 

174 # Extract coordinates from each MultiColorPeak 

175 bbox = footprint.getBBox() 

176 

177 # Create the data array from the masked images 

178 images = mExposure.image[:, bbox].array 

179 

180 # Use the inverse variance as the weights 

181 if config.useWeights: 

182 weights = 1/mExposure.variance[:, bbox].array 

183 else: 

184 weights = np.ones_like(images) 

185 badPixels = mExposure.mask.getPlaneBitMask(config.badMask) 

186 mask = mExposure.mask[:, bbox].array & badPixels 

187 weights[mask > 0] = 0 

188 

189 # Mask out the pixels outside the footprint 

190 weights *= footprint.spans.asArray() 

191 

192 psfCenter = footprint.getCentroid() 

193 psfs = mExposure.computePsfKernelImage(psfCenter).astype(np.float32) 

194 psfs = ImagePSF(psfs) 

195 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters)) 

196 

197 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters) 

198 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters) 

199 if config.convolutionType == "fft": 

200 observation.match(frame) 

201 elif config.convolutionType == "real": 

202 renderer = ConvolutionRenderer(observation, frame, convolution_type="real") 

203 observation.match(frame, renderer=renderer) 

204 else: 

205 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType)) 

206 

207 assert config.sourceModel in ["single", "double", "compact", "fit"] 

208 

209 # Set the appropriate number of components 

210 if config.sourceModel == "single": 

211 maxComponents = 1 

212 elif config.sourceModel == "double": 

213 maxComponents = 2 

214 elif config.sourceModel == "compact": 

215 maxComponents = 0 

216 elif config.sourceModel == "point": 

217 raise NotImplementedError("Point source photometry is currently not implemented") 

218 elif config.sourceModel == "fit": 

219 # It is likely in the future that there will be some heuristic 

220 # used to determine what type of model to use for each source, 

221 # but that has not yet been implemented (see DM-22551) 

222 raise NotImplementedError("sourceModel 'fit' has not been implemented yet") 

223 

224 # Convert the centers to pixel coordinates 

225 xmin = bbox.getMinX() 

226 ymin = bbox.getMinY() 

227 centers = [ 

228 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

229 for peak in footprint.peaks 

230 if not isPseudoSource(peak, config.pseudoColumns) 

231 ] 

232 

233 # Only deblend sources that can be initialized 

234 sources, skipped = init_all_sources( 

235 frame=frame, 

236 centers=centers, 

237 observations=observation, 

238 thresh=config.morphThresh, 

239 max_components=maxComponents, 

240 min_snr=config.minSNR, 

241 shifting=False, 

242 fallback=config.fallback, 

243 silent=config.catchFailures, 

244 set_spectra=spectrumInit, 

245 ) 

246 

247 # Attach the peak to all of the initialized sources 

248 srcIndex = 0 

249 for k, center in enumerate(centers): 

250 if k not in skipped: 

251 # This is just to make sure that there isn't a coding bug 

252 assert np.all(sources[srcIndex].center == center) 

253 # Store the record for the peak with the appropriate source 

254 sources[srcIndex].detectedPeak = footprint.peaks[k] 

255 srcIndex += 1 

256 

257 # Create the blend and attempt to optimize it 

258 blend = Blend(sources, observation) 

259 try: 

260 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError) 

261 except ArithmeticError: 

262 # This occurs when a gradient update produces a NaN value 

263 # This is usually due to a source initialized with a 

264 # negative SED or no flux, often because the peak 

265 # is a noise fluctuation in one band and not a real source. 

266 iterations = len(blend.loss) 

267 failedSources = [] 

268 for k, src in enumerate(sources): 

269 if np.any(~np.isfinite(src.get_model())): 

270 failedSources.append(k) 

271 raise ScarletGradientError(iterations, failedSources) 

272 

273 # Store the location of the PSF center for storage 

274 blend.psfCenter = (psfCenter.x, psfCenter.y) 

275 

276 return blend, skipped 

277 

278 

279def buildLiteObservation( 

280 modelPsf, 

281 psfCenter, 

282 mExposure, 

283 footprint=None, 

284 badPixelMasks=None, 

285 useWeights=True, 

286 convolutionType="real", 

287): 

288 """Generate a LiteObservation from a set of parameters. 

289 

290 Make the generation and reconstruction of a scarlet model consistent 

291 by building a `LiteObservation` from a set of parameters. 

292 

293 Parameters 

294 ---------- 

295 modelPsf : `numpy.ndarray` 

296 The 2D model of the PSF in the partially deconvolved space. 

297 psfCenter : `tuple` or `Point2I` or `Point2D` 

298 The location `(x, y)` used as the center of the PSF. 

299 mExposure : `lsst.afw.image.multiband.MultibandExposure` 

300 The multi-band exposure that the model represents. 

301 If `mExposure` is `None` then no image, variance, or weights are 

302 attached to the observation. 

303 footprint : `lsst.afw.detection.Footprint` 

304 The footprint that is being fit. 

305 If `Footprint` is `None` then the weights are not updated to mask 

306 out pixels not contained in the footprint. 

307 badPixelMasks : `list` of `str` 

308 The keys from the bit mask plane used to mask out pixels 

309 during the fit. 

310 If `badPixelMasks` is `None` then the default values from 

311 `ScarletDeblendConfig.badMask` is used. 

312 useWeights : `bool` 

313 Whether or not fitting should use inverse variance weights to 

314 calculate the log-likelihood. 

315 convolutionType : `str` 

316 The type of convolution to use (either "real" or "fft"). 

317 When reconstructing an image it is advised to use "real" to avoid 

318 polluting the footprint with 

319 

320 Returns 

321 ------- 

322 observation : `scarlet.lite.LiteObservation` 

323 The observation constructed from the input parameters. 

324 """ 

325 # Initialize the observed PSFs 

326 psfModels, mExposure = computePsfKernelImage(mExposure, psfCenter) 

327 

328 # Use the inverse variance as the weights 

329 if useWeights: 

330 weights = 1/mExposure.variance.array 

331 else: 

332 # Mask out bad pixels 

333 weights = np.ones_like(mExposure.image.array) 

334 if badPixelMasks is None: 

335 badPixelMasks = ScarletDeblendConfig().badMask 

336 badPixels = mExposure.mask.getPlaneBitMask(badPixelMasks) 

337 mask = mExposure.mask.array & badPixels 

338 weights[mask > 0] = 0 

339 

340 if footprint is not None: 

341 # Mask out the pixels outside the footprint 

342 weights *= footprint.spans.asArray() 

343 

344 observation = lite.LiteObservation( 

345 images=mExposure.image.array, 

346 variance=mExposure.variance.array, 

347 weights=weights, 

348 psfs=psfModels, 

349 model_psf=modelPsf[None, :, :], 

350 convolution_mode=convolutionType, 

351 ) 

352 

353 # Store the bands used to create the observation 

354 observation.bands = mExposure.filters 

355 return observation 

356 

357 

358def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None): 

359 """Deblend a parent footprint 

360 

361 Parameters 

362 ---------- 

363 mExposure : `lsst.image.MultibandExposure` 

364 - The multiband exposure containing the image, 

365 mask, and variance data 

366 footprint : `lsst.detection.Footprint` 

367 - The footprint of the parent to deblend 

368 config : `ScarletDeblendConfig` 

369 - Configuration of the deblending task 

370 

371 Returns 

372 ------- 

373 blend : `scarlet.lite.Blend` 

374 The blend this is to be deblended 

375 skippedSources : `list[int]` 

376 Indices of sources that were skipped due to no flux. 

377 This usually means that a source was a spurrious detection in one 

378 band that should not have been included in the merged catalog. 

379 skippedBands : `list[str]` 

380 Bands that were skipped because a PSF could not be generated for them. 

381 """ 

382 # Extract coordinates from each MultiColorPeak 

383 bbox = footprint.getBBox() 

384 psfCenter = footprint.getCentroid() 

385 

386 observation = buildLiteObservation( 

387 modelPsf=modelPsf, 

388 psfCenter=psfCenter, 

389 mExposure=mExposure[:, bbox], 

390 footprint=footprint, 

391 badPixelMasks=config.badMask, 

392 useWeights=config.useWeights, 

393 convolutionType=config.convolutionType, 

394 ) 

395 

396 # Convert the centers to pixel coordinates 

397 xmin = bbox.getMinX() 

398 ymin = bbox.getMinY() 

399 centers = [ 

400 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

401 for peak in footprint.peaks 

402 if not isPseudoSource(peak, config.pseudoColumns) 

403 ] 

404 

405 # Initialize the sources 

406 if config.morphImage == "chi2": 

407 sources = lite.init_all_sources_main( 

408 observation, 

409 centers, 

410 min_snr=config.minSNR, 

411 thresh=config.morphThresh, 

412 ) 

413 elif config.morphImage == "wavelet": 

414 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin()) 

415 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)] 

416 sources = lite.init_all_sources_wavelets( 

417 observation, 

418 centers, 

419 use_psf=False, 

420 wavelets=_wavelets, 

421 min_snr=config.minSNR, 

422 ) 

423 else: 

424 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.") 

425 

426 # Set the optimizer 

427 if config.optimizer == "adaprox": 

428 parameterization = partial( 

429 lite.init_adaprox_component, 

430 bg_thresh=config.backgroundThresh, 

431 max_prox_iter=config.maxProxIter, 

432 ) 

433 elif config.optimizer == "fista": 

434 parameterization = partial( 

435 lite.init_fista_component, 

436 bg_thresh=config.backgroundThresh, 

437 ) 

438 else: 

439 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.") 

440 sources = lite.parameterize_sources(sources, observation, parameterization) 

441 

442 # Attach the peak to all of the initialized sources 

443 for k, center in enumerate(centers): 

444 # This is just to make sure that there isn't a coding bug 

445 if len(sources[k].components) > 0 and np.any(sources[k].center != center): 

446 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}") 

447 # Store the record for the peak with the appropriate source 

448 sources[k].detectedPeak = footprint.peaks[k] 

449 

450 blend = lite.LiteBlend(sources, observation) 

451 

452 # Initialize each source with its best fit spectrum 

453 if spectrumInit: 

454 blend.fit_spectra() 

455 

456 # Set the sources that could not be initialized and were skipped 

457 skippedSources = [src for src in sources if src.is_null] 

458 

459 blend.fit( 

460 max_iter=config.maxIter, 

461 e_rel=config.relativeError, 

462 min_iter=config.minIter, 

463 reweight=False, 

464 ) 

465 

466 # Store the location of the PSF center for storage 

467 blend.psfCenter = (psfCenter.x, psfCenter.y) 

468 

469 # Calculate the bands that were skipped 

470 skippedBands = [band for band in mExposure.filters if band not in observation.bands] 

471 

472 return blend, skippedSources, skippedBands 

473 

474 

475@dataclass 

476class DeblenderMetrics: 

477 """Metrics and measurements made on single sources. 

478 

479 Store deblender metrics to be added as attributes to a scarlet source 

480 before it is converted into a `SourceRecord`. 

481 When DM-34414 is finished this class will be eliminated and the metrics 

482 will be added to the schema using a pipeline task that calculates them 

483 from the stored deconvolved models. 

484 

485 All of the parameters are one dimensional numpy arrays, 

486 with an element for each band in the observed images. 

487 

488 `maxOverlap` is useful as a metric for determining how blended a source 

489 is because if it only overlaps with other sources at or below 

490 the noise level, it is likely to be a mostly isolated source 

491 in the deconvolved model frame. 

492 

493 `fluxOverlapFraction` is potentially more useful than the canonical 

494 "blendedness" (or purity) metric because it accounts for potential 

495 biases created during deblending by not weighting the overlapping 

496 flux with the flux of this sources model. 

497 

498 Attributes 

499 ---------- 

500 maxOverlap : `numpy.ndarray` 

501 The maximum overlap that the source has with its neighbors in 

502 a single pixel. 

503 fluxOverlap : `numpy.ndarray` 

504 The total flux from neighbors overlapping with the current source. 

505 fluxOverlapFraction : `numpy.ndarray` 

506 The fraction of `flux from neighbors/source flux` for a 

507 given source within the source's footprint. 

508 blendedness : `numpy.ndarray` 

509 The metric for determining how blended a source is using the 

510 Bosch et al. 2018 metric for "blendedness." Note that some 

511 surveys use the term "purity," which is `1-blendedness`. 

512 """ 

513 maxOverlap: np.array 

514 fluxOverlap: np.array 

515 fluxOverlapFraction: np.array 

516 blendedness: np.array 

517 

518 

519def setDeblenderMetrics(blend): 

520 """Set metrics that can be used to evalute the deblender accuracy 

521 

522 This function calculates the `DeblenderMetrics` for each source in the 

523 blend, and assigns it to that sources `metrics` property in place. 

524 

525 Parameters 

526 ---------- 

527 blend : `scarlet.lite.Blend` 

528 The blend containing the sources to measure. 

529 """ 

530 # Store the full model of the scene for comparison 

531 blendModel = blend.get_model() 

532 for k, src in enumerate(blend.sources): 

533 # Extract the source model in the full bounding box 

534 model = src.get_model(bbox=blend.bbox) 

535 # The footprint is the 2D array of non-zero pixels in each band 

536 footprint = np.bitwise_or.reduce(model > 0, axis=0) 

537 # Calculate the metrics. 

538 # See `DeblenderMetrics` for a description of each metric. 

539 neighborOverlap = (blendModel-model) * footprint[None, :, :] 

540 maxOverlap = np.max(neighborOverlap, axis=(1, 2)) 

541 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2)) 

542 fluxModel = np.sum(model, axis=(1, 2)) 

543 fluxOverlapFraction = np.zeros((len(model), ), dtype=float) 

544 isFinite = fluxModel > 0 

545 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite] 

546 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2)) 

547 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness) 

548 

549 

550class ScarletDeblendConfig(pexConfig.Config): 

551 """MultibandDeblendConfig 

552 

553 Configuration for the multiband deblender. 

554 The parameters are organized by the parameter types, which are 

555 - Stopping Criteria: Used to determine if the fit has converged 

556 - Position Fitting Criteria: Used to fit the positions of the peaks 

557 - Constraints: Used to apply constraints to the peaks and their components 

558 - Other: Parameters that don't fit into the above categories 

559 """ 

560 # Stopping Criteria 

561 minIter = pexConfig.Field(dtype=int, default=1, 

562 doc="Minimum number of iterations before the optimizer is allowed to stop.") 

563 maxIter = pexConfig.Field(dtype=int, default=300, 

564 doc=("Maximum number of iterations to deblend a single parent")) 

565 relativeError = pexConfig.Field(dtype=float, default=1e-2, 

566 doc=("Change in the loss function between iterations to exit fitter. " 

567 "Typically this is `1e-2` if measurements will be made on the " 

568 "flux re-distributed models and `1e-4` when making measurements " 

569 "on the models themselves.")) 

570 

571 # Constraints 

572 morphThresh = pexConfig.Field(dtype=float, default=1, 

573 doc="Fraction of background RMS a pixel must have" 

574 "to be included in the initial morphology") 

575 # Lite Parameters 

576 # All of these parameters (except version) are only valid if version='lite' 

577 version = pexConfig.ChoiceField( 

578 dtype=str, 

579 default="lite", 

580 allowed={ 

581 "scarlet": "main scarlet version (likely to be deprecated soon)", 

582 "lite": "Optimized version of scarlet for survey data from a single instrument", 

583 }, 

584 doc="The version of scarlet to use.", 

585 ) 

586 optimizer = pexConfig.ChoiceField( 

587 dtype=str, 

588 default="adaprox", 

589 allowed={ 

590 "adaprox": "Proximal ADAM optimization", 

591 "fista": "Accelerated proximal gradient method", 

592 }, 

593 doc="The optimizer to use for fitting parameters and is only used when version='lite'", 

594 ) 

595 morphImage = pexConfig.ChoiceField( 

596 dtype=str, 

597 default="chi2", 

598 allowed={ 

599 "chi2": "Initialize sources on a chi^2 image made from all available bands", 

600 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image", 

601 }, 

602 doc="The type of image to use for initializing the morphology. " 

603 "Must be either 'chi2' or 'wavelet'. " 

604 ) 

605 backgroundThresh = pexConfig.Field( 

606 dtype=float, 

607 default=0.25, 

608 doc="Fraction of background to use for a sparsity threshold. " 

609 "This prevents sources from growing unrealistically outside " 

610 "the parent footprint while still modeling flux correctly " 

611 "for bright sources." 

612 ) 

613 maxProxIter = pexConfig.Field( 

614 dtype=int, 

615 default=1, 

616 doc="Maximum number of proximal operator iterations inside of each " 

617 "iteration of the optimizer. " 

618 "This config field is only used if version='lite' and optimizer='adaprox'." 

619 ) 

620 waveletScales = pexConfig.Field( 

621 dtype=int, 

622 default=5, 

623 doc="Number of wavelet scales to use for wavelet initialization. " 

624 "This field is only used when `version`='lite' and `morphImage`='wavelet'." 

625 ) 

626 

627 # Other scarlet paremeters 

628 useWeights = pexConfig.Field( 

629 dtype=bool, default=True, 

630 doc=("Whether or not use use inverse variance weighting." 

631 "If `useWeights` is `False` then flat weights are used")) 

632 modelPsfSize = pexConfig.Field( 

633 dtype=int, default=11, 

634 doc="Model PSF side length in pixels") 

635 modelPsfSigma = pexConfig.Field( 

636 dtype=float, default=0.8, 

637 doc="Define sigma for the model frame PSF") 

638 minSNR = pexConfig.Field( 

639 dtype=float, default=50, 

640 doc="Minimum Signal to noise to accept the source." 

641 "Sources with lower flux will be initialized with the PSF but updated " 

642 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).") 

643 saveTemplates = pexConfig.Field( 

644 dtype=bool, default=True, 

645 doc="Whether or not to save the SEDs and templates") 

646 processSingles = pexConfig.Field( 

647 dtype=bool, default=True, 

648 doc="Whether or not to process isolated sources in the deblender") 

649 convolutionType = pexConfig.Field( 

650 dtype=str, default="fft", 

651 doc="Type of convolution to render the model to the observations.\n" 

652 "- 'fft': perform convolutions in Fourier space\n" 

653 "- 'real': peform convolutions in real space.") 

654 sourceModel = pexConfig.Field( 

655 dtype=str, default="double", 

656 doc=("How to determine which model to use for sources, from\n" 

657 "- 'single': use a single component for all sources\n" 

658 "- 'double': use a bulge disk model for all sources\n" 

659 "- 'compact': use a single component model, initialzed with a point source morphology, " 

660 " for all sources\n" 

661 "- 'point': use a point-source model for all sources\n" 

662 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"), 

663 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.", 

664 ) 

665 setSpectra = pexConfig.Field( 

666 dtype=bool, default=True, 

667 doc="Whether or not to solve for the best-fit spectra during initialization. " 

668 "This makes initialization slightly longer, as it requires a convolution " 

669 "to set the optimal spectra, but results in a much better initial log-likelihood " 

670 "and reduced total runtime, with convergence in fewer iterations." 

671 "This option is only used when " 

672 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.") 

673 

674 # Mask-plane restrictions 

675 badMask = pexConfig.ListField( 

676 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"], 

677 doc="Whether or not to process isolated sources in the deblender") 

678 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

679 doc="Mask planes to ignore when performing statistics") 

680 maskLimits = pexConfig.DictField( 

681 keytype=str, 

682 itemtype=float, 

683 default={}, 

684 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

685 "Sources violating this limit will not be deblended. " 

686 "If the fraction is `0` then the limit is a single pixel."), 

687 ) 

688 

689 # Size restrictions 

690 maxNumberOfPeaks = pexConfig.Field( 

691 dtype=int, default=200, 

692 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

693 " (<= 0: unlimited)")) 

694 maxFootprintArea = pexConfig.Field( 

695 dtype=int, default=100_000, 

696 doc=("Maximum area for footprints before they are ignored as large; " 

697 "non-positive means no threshold applied")) 

698 maxAreaTimesPeaks = pexConfig.Field( 

699 dtype=int, default=10_000_000, 

700 doc=("Maximum rectangular footprint area * nPeaks in the footprint. " 

701 "This was introduced in DM-33690 to prevent fields that are crowded or have a " 

702 "LSB galaxy that causes memory intensive initialization in scarlet from dominating " 

703 "the overall runtime and/or causing the task to run out of memory. " 

704 "(<= 0: unlimited)") 

705 ) 

706 maxFootprintSize = pexConfig.Field( 

707 dtype=int, default=0, 

708 doc=("Maximum linear dimension for footprints before they are ignored " 

709 "as large; non-positive means no threshold applied")) 

710 minFootprintAxisRatio = pexConfig.Field( 

711 dtype=float, default=0.0, 

712 doc=("Minimum axis ratio for footprints before they are ignored " 

713 "as large; non-positive means no threshold applied")) 

714 maxSpectrumCutoff = pexConfig.Field( 

715 dtype=int, default=1_000_000, 

716 doc=("Maximum number of pixels * number of sources in a blend. " 

717 "This is different than `maxFootprintArea` because this isn't " 

718 "the footprint area but the area of the bounding box that " 

719 "contains the footprint, and is also multiplied by the number of" 

720 "sources in the footprint. This prevents large skinny blends with " 

721 "a high density of sources from running out of memory. " 

722 "If `maxSpectrumCutoff == -1` then there is no cutoff.") 

723 ) 

724 # Failure modes 

725 fallback = pexConfig.Field( 

726 dtype=bool, default=True, 

727 doc="Whether or not to fallback to a smaller number of components if a source does not initialize" 

728 ) 

729 notDeblendedMask = pexConfig.Field( 

730 dtype=str, default="NOT_DEBLENDED", optional=True, 

731 doc="Mask name for footprints not deblended, or None") 

732 catchFailures = pexConfig.Field( 

733 dtype=bool, default=True, 

734 doc=("If True, catch exceptions thrown by the deblender, log them, " 

735 "and set a flag on the parent, instead of letting them propagate up")) 

736 

737 # Other options 

738 columnInheritance = pexConfig.DictField( 

739 keytype=str, itemtype=str, default={ 

740 "deblend_nChild": "deblend_parentNChild", 

741 "deblend_nPeaks": "deblend_parentNPeaks", 

742 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag", 

743 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag", 

744 }, 

745 doc="Columns to pass from the parent to the child. " 

746 "The key is the name of the column for the parent record, " 

747 "the value is the name of the column to use for the child." 

748 ) 

749 pseudoColumns = pexConfig.ListField( 

750 dtype=str, default=['merge_peak_sky', 'sky_source'], 

751 doc="Names of flags which should never be deblended." 

752 ) 

753 

754 # Logging option(s) 

755 loggingInterval = pexConfig.Field( 

756 dtype=int, default=600, 

757 doc="Interval (in seconds) to log messages (at VERBOSE level) while deblending sources.", 

758 deprecated="This field is no longer used and will be removed in v25.", 

759 ) 

760 # Testing options 

761 # Some obs packages and ci packages run the full pipeline on a small 

762 # subset of data to test that the pipeline is functioning properly. 

763 # This is not meant as scientific validation, so it can be useful 

764 # to only run on a small subset of the data that is large enough to 

765 # test the desired pipeline features but not so long that the deblender 

766 # is the tall pole in terms of execution times. 

767 useCiLimits = pexConfig.Field( 

768 dtype=bool, default=False, 

769 doc="Limit the number of sources deblended for CI to prevent long build times") 

770 ciDeblendChildRange = pexConfig.ListField( 

771 dtype=int, default=[5, 10], 

772 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated." 

773 "If `useCiLimits==False` then this parameter is ignored.") 

774 ciNumParentsToDeblend = pexConfig.Field( 

775 dtype=int, default=10, 

776 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count " 

777 "within `ciDebledChildRange`. " 

778 "If `useCiLimits==False` then this parameter is ignored.") 

779 

780 

781class ScarletDeblendTask(pipeBase.Task): 

782 """ScarletDeblendTask 

783 

784 Split blended sources into individual sources. 

785 

786 This task has no return value; it only modifies the SourceCatalog in-place. 

787 """ 

788 ConfigClass = ScarletDeblendConfig 

789 _DefaultName = "scarletDeblend" 

790 

791 def __init__(self, schema, peakSchema=None, **kwargs): 

792 """Create the task, adding necessary fields to the given schema. 

793 

794 Parameters 

795 ---------- 

796 schema : `lsst.afw.table.schema.schema.Schema` 

797 Schema object for measurement fields; will be modified in-place. 

798 peakSchema : `lsst.afw.table.schema.schema.Schema` 

799 Schema of Footprint Peaks that will be passed to the deblender. 

800 Any fields beyond the PeakTable minimal schema will be transferred 

801 to the main source Schema. If None, no fields will be transferred 

802 from the Peaks. 

803 filters : list of str 

804 Names of the filters used for the eposures. This is needed to store 

805 the SED as a field 

806 **kwargs 

807 Passed to Task.__init__. 

808 """ 

809 pipeBase.Task.__init__(self, **kwargs) 

810 

811 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

812 if peakSchema is None: 

813 # In this case, the peakSchemaMapper will transfer nothing, but 

814 # we'll still have one 

815 # to simplify downstream code 

816 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

817 else: 

818 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

819 for item in peakSchema: 

820 if item.key not in peakMinimalSchema: 

821 self.peakSchemaMapper.addMapping(item.key, item.field) 

822 # Because SchemaMapper makes a copy of the output schema 

823 # you give its ctor, it isn't updating this Schema in 

824 # place. That's probably a design flaw, but in the 

825 # meantime, we'll keep that schema in sync with the 

826 # peakSchemaMapper.getOutputSchema() manually, by adding 

827 # the same fields to both. 

828 schema.addField(item.field) 

829 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

830 self._addSchemaKeys(schema) 

831 self.schema = schema 

832 self.toCopyFromParent = [item.key for item in self.schema 

833 if item.field.getName().startswith("merge_footprint")] 

834 

835 def _addSchemaKeys(self, schema): 

836 """Add deblender specific keys to the schema 

837 """ 

838 # Parent (blend) fields 

839 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms') 

840 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge') 

841 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

842 doc='Number of children this object has (defaults to 0)') 

843 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32, 

844 doc="Number of initial peaks in the blend. " 

845 "This includes peaks that may have been culled " 

846 "during deblending or failed to deblend") 

847 # Skipped flags 

848 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

849 doc="Deblender skipped this source") 

850 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag', 

851 doc='The source has only a single peak ' 

852 'and was not deblended') 

853 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag', 

854 doc='The source is identified as a "pseudo" source and ' 

855 'was not deblended') 

856 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

857 doc='Source had too many peaks; ' 

858 'only the brightest were included') 

859 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

860 doc='Parent footprint covered too many pixels') 

861 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

862 doc='Parent footprint had too many masked pixels') 

863 # Convergence flags 

864 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag', 

865 doc='scarlet sed optimization did not converge before' 

866 'config.maxIter') 

867 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag', 

868 doc='scarlet morph optimization did not converge before' 

869 'config.maxIter') 

870 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag', 

871 type='Flag', 

872 doc='at least one source in the blend' 

873 'failed to converge') 

874 # Error flags 

875 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

876 doc="Deblending failed on source") 

877 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25, 

878 doc='Name of error if the blend failed') 

879 self.incompleteDataKey = schema.addField('deblend_incompleteData', type='Flag', 

880 doc='True when a blend has at least one band ' 

881 'that could not generate a PSF and was ' 

882 'not included in the model.') 

883 # Deblended source fields 

884 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center", 

885 doc="Center used to apply constraints in scarlet", 

886 unit="pixel") 

887 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32, 

888 doc="ID of the peak in the parent footprint. " 

889 "This is not unique, but the combination of 'parent'" 

890 "and 'peakId' should be for all child sources. " 

891 "Top level blends with no parents have 'peakId=0'") 

892 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count', 

893 doc="The instFlux at the peak position of deblended mode") 

894 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25, 

895 doc="The type of model used, for example " 

896 "MultiExtendedSource, SingleExtendedSource, PointSource") 

897 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32, 

898 doc="deblend_nPeaks from this records parent.") 

899 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32, 

900 doc="deblend_nChild from this records parent.") 

901 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32, 

902 doc="Flux measurement from scarlet") 

903 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32, 

904 doc="Final logL, used to identify regressions in scarlet.") 

905 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag', 

906 doc='Source had flux on the edge of the parent footprint') 

907 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag', 

908 doc="True when scarlet initializes sources " 

909 "in the blend with a more accurate spectrum. " 

910 "The algorithm uses a lot of memory, " 

911 "so large dense blends will use " 

912 "a less accurate initialization.") 

913 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32, 

914 doc="Number of components in a ScarletLiteSource. " 

915 "If `config.version != 'lite'`then " 

916 "this column is set to zero.") 

917 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

918 doc='Deblender thought this source looked like a PSF') 

919 self.coverageKey = schema.addField('deblend_dataCoverage', type=np.float32, 

920 doc='Fraction of pixels with data. ' 

921 'In other words, 1 - fraction of pixels with NO_DATA set.') 

922 # Blendedness/classification metrics 

923 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32, 

924 doc="Maximum overlap with all of the other neighbors flux " 

925 "combined." 

926 "This is useful as a metric for determining how blended a " 

927 "source is because if it only overlaps with other sources " 

928 "at or below the noise level, it is likely to be a mostly " 

929 "isolated source in the deconvolved model frame.") 

930 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32, 

931 doc="This is the total flux from neighboring objects that " 

932 "overlaps with this source.") 

933 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32, 

934 doc="This is the fraction of " 

935 "`flux from neighbors/source flux` " 

936 "for a given source within the source's" 

937 "footprint.") 

938 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32, 

939 doc="The Bosch et al. 2018 metric for 'blendedness.' ") 

940 

941 @timeMethod 

942 def run(self, mExposure, mergedSources): 

943 """Get the psf from each exposure and then run deblend(). 

944 

945 Parameters 

946 ---------- 

947 mExposure : `MultibandExposure` 

948 The exposures should be co-added images of the same 

949 shape and region of the sky. 

950 mergedSources : `SourceCatalog` 

951 The merged `SourceCatalog` that contains parent footprints 

952 to (potentially) deblend. 

953 

954 Returns 

955 ------- 

956 templateCatalogs: dict 

957 Keys are the names of the filters and the values are 

958 `lsst.afw.table.source.source.SourceCatalog`'s. 

959 These are catalogs with heavy footprints that are the templates 

960 created by the multiband templates. 

961 """ 

962 return self.deblend(mExposure, mergedSources) 

963 

964 @timeMethod 

965 def deblend(self, mExposure, catalog): 

966 """Deblend a data cube of multiband images 

967 

968 Parameters 

969 ---------- 

970 mExposure : `MultibandExposure` 

971 The exposures should be co-added images of the same 

972 shape and region of the sky. 

973 catalog : `SourceCatalog` 

974 The merged `SourceCatalog` that contains parent footprints 

975 to (potentially) deblend. The new deblended sources are 

976 appended to this catalog in place. 

977 

978 Returns 

979 ------- 

980 catalogs : `dict` or `None` 

981 Keys are the names of the filters and the values are 

982 `lsst.afw.table.source.source.SourceCatalog`'s. 

983 These are catalogs with heavy footprints that are the templates 

984 created by the multiband templates. 

985 """ 

986 import time 

987 

988 # Cull footprints if required by ci 

989 if self.config.useCiLimits: 

990 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.", 

991 len(catalog)) 

992 # Select parents with a number of children in the range 

993 # config.ciDeblendChildRange 

994 minChildren, maxChildren = self.config.ciDeblendChildRange 

995 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog]) 

996 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0] 

997 if len(childrenInRange) < self.config.ciNumParentsToDeblend: 

998 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range " 

999 "indicated by ciDeblendChildRange. Adjust this range to include more " 

1000 "parents.") 

1001 # Keep all of the isolated parents and the first 

1002 # `ciNumParentsToDeblend` children 

1003 parents = nPeaks == 1 

1004 children = np.zeros((len(catalog),), dtype=bool) 

1005 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True 

1006 catalog = catalog[parents | children] 

1007 # We need to update the IdFactory, otherwise the the source ids 

1008 # will not be sequential 

1009 idFactory = catalog.getIdFactory() 

1010 maxId = np.max(catalog["id"]) 

1011 idFactory.notify(maxId) 

1012 

1013 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure)) 

1014 periodicLog = PeriodicLogger(self.log) 

1015 

1016 # Create a set of wavelet coefficients if using wavelet initialization 

1017 if self.config.version == "lite" and self.config.morphImage == "wavelet": 

1018 images = mExposure.image.array 

1019 variance = mExposure.variance.array 

1020 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales) 

1021 else: 

1022 wavelets = None 

1023 

1024 # Add the NOT_DEBLENDED mask to the mask plane in each band 

1025 if self.config.notDeblendedMask: 

1026 for mask in mExposure.mask: 

1027 mask.addMaskPlane(self.config.notDeblendedMask) 

1028 

1029 # Initialize the persistable data model 

1030 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma) 

1031 dataModel = ScarletModelData(modelPsf) 

1032 

1033 nParents = len(catalog) 

1034 nDeblendedParents = 0 

1035 skippedParents = [] 

1036 for parentIndex in range(nParents): 

1037 parent = catalog[parentIndex] 

1038 foot = parent.getFootprint() 

1039 bbox = foot.getBBox() 

1040 peaks = foot.getPeaks() 

1041 

1042 # Since we use the first peak for the parent object, we should 

1043 # propagate its flags to the parent source. 

1044 parent.assign(peaks[0], self.peakSchemaMapper) 

1045 

1046 # Block of conditions for skipping a parent with multiple children 

1047 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None: 

1048 self._skipParent(parent, *skipArgs) 

1049 skippedParents.append(parentIndex) 

1050 continue 

1051 

1052 nDeblendedParents += 1 

1053 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks)) 

1054 # Run the deblender 

1055 blendError = None 

1056 

1057 # Choose whether or not to use improved spectral initialization. 

1058 # This significantly cuts down on the number of iterations 

1059 # that the optimizer needs and usually results in a better 

1060 # fit. 

1061 # But using least squares on a very large blend causes memory 

1062 # issues, so it is not done for large blends 

1063 if self.config.setSpectra: 

1064 if self.config.maxSpectrumCutoff <= 0: 

1065 spectrumInit = True 

1066 else: 

1067 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff 

1068 else: 

1069 spectrumInit = False 

1070 

1071 try: 

1072 t0 = time.monotonic() 

1073 # Build the parameter lists with the same ordering 

1074 if self.config.version == "scarlet": 

1075 blend, skippedSources = deblend(mExposure, foot, self.config, spectrumInit) 

1076 skippedBands = [] 

1077 elif self.config.version == "lite": 

1078 blend, skippedSources, skippedBands = deblend_lite( 

1079 mExposure=mExposure, 

1080 modelPsf=modelPsf, 

1081 footprint=foot, 

1082 config=self.config, 

1083 spectrumInit=spectrumInit, 

1084 wavelets=wavelets, 

1085 ) 

1086 tf = time.monotonic() 

1087 runtime = (tf-t0)*1000 

1088 converged = _checkBlendConvergence(blend, self.config.relativeError) 

1089 # Store the number of components in the blend 

1090 if self.config.version == "lite": 

1091 nComponents = len(blend.components) 

1092 else: 

1093 nComponents = 0 

1094 nChild = len(blend.sources) 

1095 parent.set(self.incompleteDataKey, len(skippedBands) > 0) 

1096 # Catch all errors and filter out the ones that we know about 

1097 except Exception as e: 

1098 print("deblend failed") 

1099 print(e) 

1100 blendError = type(e).__name__ 

1101 if isinstance(e, ScarletGradientError): 

1102 parent.set(self.iterKey, e.iterations) 

1103 else: 

1104 blendError = "UnknownError" 

1105 if self.config.catchFailures: 

1106 # Make it easy to find UnknownErrors in the log file 

1107 self.log.warn("UnknownError") 

1108 import traceback 

1109 traceback.print_exc() 

1110 else: 

1111 raise 

1112 

1113 self._skipParent( 

1114 parent=parent, 

1115 skipKey=self.deblendFailedKey, 

1116 logMessage=f"Unable to deblend source {parent.getId}: {blendError}", 

1117 ) 

1118 parent.set(self.deblendErrorKey, blendError) 

1119 skippedParents.append(parentIndex) 

1120 continue 

1121 

1122 # Update the parent record with the deblending results 

1123 if self.config.version == "scarlet": 

1124 logL = -blend.loss[-1] + blend.observations[0].log_norm 

1125 elif self.config.version == "lite": 

1126 logL = blend.loss[-1] 

1127 self._updateParentRecord( 

1128 parent=parent, 

1129 nPeaks=len(peaks), 

1130 nChild=nChild, 

1131 nComponents=nComponents, 

1132 runtime=runtime, 

1133 iterations=len(blend.loss), 

1134 logL=logL, 

1135 spectrumInit=spectrumInit, 

1136 converged=converged, 

1137 ) 

1138 

1139 # Add each deblended source to the catalog 

1140 for k, scarletSource in enumerate(blend.sources): 

1141 # Skip any sources with no flux or that scarlet skipped because 

1142 # it could not initialize 

1143 if k in skippedSources or (self.config.version == "lite" and scarletSource.is_null): 

1144 # No need to propagate anything 

1145 continue 

1146 parent.set(self.deblendSkippedKey, False) 

1147 

1148 # Add all fields except the HeavyFootprint to the 

1149 # source record 

1150 sourceRecord = self._addChild( 

1151 parent=parent, 

1152 peak=scarletSource.detectedPeak, 

1153 catalog=catalog, 

1154 scarletSource=scarletSource, 

1155 ) 

1156 scarletSource.recordId = sourceRecord.getId() 

1157 scarletSource.peakId = scarletSource.detectedPeak.getId() 

1158 

1159 # Store the blend information so that it can be persisted 

1160 if self.config.version == "lite": 

1161 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin(), blend.observation.bands) 

1162 else: 

1163 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin(), mExposure.filters) 

1164 dataModel.blends[parent.getId()] = blendData 

1165 

1166 # Log a message if it has been a while since the last log. 

1167 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents) 

1168 

1169 # Clear the cached values in scarlet to clear out memory 

1170 scarlet.cache.Cache._cache = {} 

1171 

1172 # Update the mExposure mask with the footprint of skipped parents 

1173 if self.config.notDeblendedMask: 

1174 for mask in mExposure.mask: 

1175 for parentIndex in skippedParents: 

1176 fp = catalog[parentIndex].getFootprint() 

1177 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask)) 

1178 

1179 self.log.info("Deblender results: of %d parent sources, %d were deblended, " 

1180 "creating %d children, for a total of %d sources", 

1181 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog)) 

1182 return catalog, dataModel 

1183 

1184 def _isLargeFootprint(self, footprint): 

1185 """Returns whether a Footprint is large 

1186 

1187 'Large' is defined by thresholds on the area, size and axis ratio, 

1188 and total area of the bounding box multiplied by 

1189 the number of children. 

1190 These may be disabled independently by configuring them to be 

1191 non-positive. 

1192 """ 

1193 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

1194 return True 

1195 if self.config.maxFootprintSize > 0: 

1196 bbox = footprint.getBBox() 

1197 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

1198 return True 

1199 if self.config.minFootprintAxisRatio > 0: 

1200 axes = afwEll.Axes(footprint.getShape()) 

1201 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

1202 return True 

1203 if self.config.maxAreaTimesPeaks > 0: 

1204 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks: 

1205 return True 

1206 return False 

1207 

1208 def _isMasked(self, footprint, mExposure): 

1209 """Returns whether the footprint violates the mask limits 

1210 

1211 Parameters 

1212 ---------- 

1213 footprint : `lsst.afw.detection.Footprint` 

1214 The footprint to check for masked pixels 

1215 mMask : `lsst.afw.image.MaskX` 

1216 The mask plane to check for masked pixels in the `footprint`. 

1217 

1218 Returns 

1219 ------- 

1220 isMasked : `bool` 

1221 `True` if `self.config.maskPlaneLimits` is less than the 

1222 fraction of pixels for a given mask in 

1223 `self.config.maskLimits`. 

1224 """ 

1225 bbox = footprint.getBBox() 

1226 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0) 

1227 size = float(footprint.getArea()) 

1228 for maskName, limit in self.config.maskLimits.items(): 

1229 maskVal = mExposure.mask.getPlaneBitMask(maskName) 

1230 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin()) 

1231 # spanset of masked pixels 

1232 maskedSpan = footprint.spans.intersect(_mask, maskVal) 

1233 if (maskedSpan.getArea())/size > limit: 

1234 return True 

1235 return False 

1236 

1237 def _skipParent(self, parent, skipKey, logMessage): 

1238 """Update a parent record that is not being deblended. 

1239 

1240 This is a fairly trivial function but is implemented to ensure 

1241 that a skipped parent updates the appropriate columns 

1242 consistently, and always has a flag to mark the reason that 

1243 it is being skipped. 

1244 

1245 Parameters 

1246 ---------- 

1247 parent : `lsst.afw.table.source.source.SourceRecord` 

1248 The parent record to flag as skipped. 

1249 skipKey : `bool` 

1250 The name of the flag to mark the reason for skipping. 

1251 logMessage : `str` 

1252 The message to display in a log.trace when a source 

1253 is skipped. 

1254 """ 

1255 if logMessage is not None: 

1256 self.log.trace(logMessage) 

1257 self._updateParentRecord( 

1258 parent=parent, 

1259 nPeaks=len(parent.getFootprint().peaks), 

1260 nChild=0, 

1261 nComponents=0, 

1262 runtime=np.nan, 

1263 iterations=0, 

1264 logL=np.nan, 

1265 spectrumInit=False, 

1266 converged=False, 

1267 ) 

1268 

1269 # Mark the source as skipped by the deblender and 

1270 # flag the reason why. 

1271 parent.set(self.deblendSkippedKey, True) 

1272 parent.set(skipKey, True) 

1273 

1274 def _checkSkipped(self, parent, mExposure): 

1275 """Update a parent record that is not being deblended. 

1276 

1277 This is a fairly trivial function but is implemented to ensure 

1278 that a skipped parent updates the appropriate columns 

1279 consistently, and always has a flag to mark the reason that 

1280 it is being skipped. 

1281 

1282 Parameters 

1283 ---------- 

1284 parent : `lsst.afw.table.source.source.SourceRecord` 

1285 The parent record to flag as skipped. 

1286 mExposure : `MultibandExposure` 

1287 The exposures should be co-added images of the same 

1288 shape and region of the sky. 

1289 Returns 

1290 ------- 

1291 skip: `bool` 

1292 `True` if the deblender will skip the parent 

1293 """ 

1294 skipKey = None 

1295 skipMessage = None 

1296 footprint = parent.getFootprint() 

1297 if len(footprint.peaks) < 2 and not self.config.processSingles: 

1298 # Skip isolated sources unless processSingles is turned on. 

1299 # Note: this does not flag isolated sources as skipped or 

1300 # set the NOT_DEBLENDED mask in the exposure, 

1301 # since these aren't really any skipped blends. 

1302 skipKey = self.isolatedParentKey 

1303 elif isPseudoSource(parent, self.config.pseudoColumns): 

1304 # We also skip pseudo sources, like sky objects, which 

1305 # are intended to be skipped. 

1306 skipKey = self.pseudoKey 

1307 if self._isLargeFootprint(footprint): 

1308 # The footprint is above the maximum footprint size limit 

1309 skipKey = self.tooBigKey 

1310 skipMessage = f"Parent {parent.getId()}: skipping large footprint" 

1311 elif self._isMasked(footprint, mExposure): 

1312 # The footprint exceeds the maximum number of masked pixels 

1313 skipKey = self.maskedKey 

1314 skipMessage = f"Parent {parent.getId()}: skipping masked footprint" 

1315 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks: 

1316 # Unlike meas_deblender, in scarlet we skip the entire blend 

1317 # if the number of peaks exceeds max peaks, since neglecting 

1318 # to model any peaks often results in catastrophic failure 

1319 # of scarlet to generate models for the brighter sources. 

1320 skipKey = self.tooManyPeaksKey 

1321 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks" 

1322 if skipKey is not None: 

1323 return (skipKey, skipMessage) 

1324 return None 

1325 

1326 def setSkipFlags(self, mExposure, catalog): 

1327 """Set the skip flags for all of the parent sources 

1328 

1329 This is mostly used for testing which parent sources will be deblended 

1330 and which will be skipped based on the current configuration options. 

1331 Skipped sources will have the appropriate flags set in place in the 

1332 catalog. 

1333 

1334 Parameters 

1335 ---------- 

1336 mExposure : `MultibandExposure` 

1337 The exposures should be co-added images of the same 

1338 shape and region of the sky. 

1339 catalog : `SourceCatalog` 

1340 The merged `SourceCatalog` that contains parent footprints 

1341 to (potentially) deblend. The new deblended sources are 

1342 appended to this catalog in place. 

1343 """ 

1344 for src in catalog: 

1345 if skipArgs := self._checkSkipped(src, mExposure) is not None: 

1346 self._skipParent(src, *skipArgs) 

1347 

1348 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents, 

1349 runtime, iterations, logL, spectrumInit, converged): 

1350 """Update a parent record in all of the single band catalogs. 

1351 

1352 Ensure that all locations that update a parent record, 

1353 whether it is skipped or updated after deblending, 

1354 update all of the appropriate columns. 

1355 

1356 Parameters 

1357 ---------- 

1358 parent : `lsst.afw.table.source.source.SourceRecord` 

1359 The parent record to update. 

1360 nPeaks : `int` 

1361 Number of peaks in the parent footprint. 

1362 nChild : `int` 

1363 Number of children deblended from the parent. 

1364 This may differ from `nPeaks` if some of the peaks 

1365 were culled and have no deblended model. 

1366 nComponents : `int` 

1367 Total number of components in the parent. 

1368 This is usually different than the number of children, 

1369 since it is common for a single source to have multiple 

1370 components. 

1371 runtime : `float` 

1372 Total runtime for deblending. 

1373 iterations : `int` 

1374 Total number of iterations in scarlet before convergence. 

1375 logL : `float` 

1376 Final log likelihood of the blend. 

1377 spectrumInit : `bool` 

1378 True when scarlet used `set_spectra` to initialize all 

1379 sources with better initial intensities. 

1380 converged : `bool` 

1381 True when the optimizer reached convergence before 

1382 reaching the maximum number of iterations. 

1383 """ 

1384 parent.set(self.nPeaksKey, nPeaks) 

1385 parent.set(self.nChildKey, nChild) 

1386 parent.set(self.nComponentsKey, nComponents) 

1387 parent.set(self.runtimeKey, runtime) 

1388 parent.set(self.iterKey, iterations) 

1389 parent.set(self.scarletLogLKey, logL) 

1390 parent.set(self.scarletSpectrumInitKey, spectrumInit) 

1391 parent.set(self.blendConvergenceFailedFlagKey, converged) 

1392 

1393 def _addChild(self, parent, peak, catalog, scarletSource): 

1394 """Add a child to a catalog. 

1395 

1396 This creates a new child in the source catalog, 

1397 assigning it a parent id, and adding all columns 

1398 that are independent across all filter bands. 

1399 

1400 Parameters 

1401 ---------- 

1402 parent : `lsst.afw.table.source.source.SourceRecord` 

1403 The parent of the new child record. 

1404 peak : `lsst.afw.table.PeakRecord` 

1405 The peak record for the peak from the parent peak catalog. 

1406 catalog : `lsst.afw.table.source.source.SourceCatalog` 

1407 The merged `SourceCatalog` that contains parent footprints 

1408 to (potentially) deblend. 

1409 scarletSource : `scarlet.Component` 

1410 The scarlet model for the new source record. 

1411 """ 

1412 src = catalog.addNew() 

1413 for key in self.toCopyFromParent: 

1414 src.set(key, parent.get(key)) 

1415 # The peak catalog is the same for all bands, 

1416 # so we just use the first peak catalog 

1417 src.assign(peak, self.peakSchemaMapper) 

1418 src.setParent(parent.getId()) 

1419 src.set(self.nPeaksKey, 1) 

1420 # Set the psf key based on whether or not the source was 

1421 # deblended using the PointSource model. 

1422 # This key is not that useful anymore since we now keep track of 

1423 # `modelType`, but we continue to propagate it in case code downstream 

1424 # is expecting it. 

1425 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource") 

1426 src.set(self.modelTypeKey, scarletSource.__class__.__name__) 

1427 # We set the runtime to zero so that summing up the 

1428 # runtime column will give the total time spent 

1429 # running the deblender for the catalog. 

1430 src.set(self.runtimeKey, 0) 

1431 

1432 # Set the position of the peak from the parent footprint 

1433 # This will make it easier to match the same source across 

1434 # deblenders and across observations, where the peak 

1435 # position is unlikely to change unless enough time passes 

1436 # for a source to move on the sky. 

1437 src.set(self.peakCenter, geom.Point2I(peak["i_x"], peak["i_y"])) 

1438 src.set(self.peakIdKey, peak["id"]) 

1439 

1440 # Store the number of components for the source 

1441 src.set(self.nComponentsKey, len(scarletSource.components)) 

1442 

1443 # Flag sources missing one or more bands 

1444 src.set(self.incompleteDataKey, parent.get(self.incompleteDataKey)) 

1445 

1446 # Propagate columns from the parent to the child 

1447 for parentColumn, childColumn in self.config.columnInheritance.items(): 

1448 src.set(childColumn, parent.get(parentColumn)) 

1449 

1450 return src