Coverage for python/lsst/meas/extensions/scarlet/scarletDeblendTask.py: 17%

470 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-03 04:12 -0700

1# This file is part of meas_extensions_scarlet. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from dataclasses import dataclass 

23from functools import partial 

24import logging 

25import numpy as np 

26import scarlet 

27from scarlet.psf import ImagePSF, GaussianPSF 

28from scarlet import Blend, Frame, Observation 

29from scarlet.renderer import ConvolutionRenderer 

30from scarlet.detect import get_detect_wavelets 

31from scarlet.initialization import init_all_sources 

32from scarlet import lite 

33 

34import lsst.pex.config as pexConfig 

35from lsst.pex.exceptions import InvalidParameterError 

36import lsst.pipe.base as pipeBase 

37from lsst.geom import Point2I, Box2I, Point2D 

38import lsst.afw.geom.ellipses as afwEll 

39import lsst.afw.image as afwImage 

40import lsst.afw.detection as afwDet 

41import lsst.afw.table as afwTable 

42from lsst.utils.logging import PeriodicLogger 

43from lsst.utils.timer import timeMethod 

44 

45from .source import bboxToScarletBox 

46from .io import ScarletModelData, scarletToData, scarletLiteToData 

47 

48# Scarlet and proxmin have a different definition of log levels than the stack, 

49# so even "warnings" occur far more often than we would like. 

50# So for now we only display scarlet and proxmin errors, as all other 

51# scarlet outputs would be considered "TRACE" by our standards. 

52scarletLogger = logging.getLogger("scarlet") 

53scarletLogger.setLevel(logging.ERROR) 

54proxminLogger = logging.getLogger("proxmin") 

55proxminLogger.setLevel(logging.ERROR) 

56 

57__all__ = ["deblend", "ScarletDeblendConfig", "ScarletDeblendTask"] 

58 

59logger = logging.getLogger(__name__) 

60 

61 

62class IncompleteDataError(Exception): 

63 """The PSF could not be computed due to incomplete data 

64 """ 

65 pass 

66 

67 

68class ScarletGradientError(Exception): 

69 """An error occurred during optimization 

70 

71 This error occurs when the optimizer encounters 

72 a NaN value while calculating the gradient. 

73 """ 

74 def __init__(self, iterations, sources): 

75 self.iterations = iterations 

76 self.sources = sources 

77 msg = ("ScalarGradientError in iteration {0}. " 

78 "NaN values introduced in sources {1}") 

79 self.message = msg.format(iterations, sources) 

80 

81 def __str__(self): 

82 return self.message 

83 

84 

85def _checkBlendConvergence(blend, f_rel): 

86 """Check whether or not a blend has converged 

87 """ 

88 deltaLoss = np.abs(blend.loss[-2] - blend.loss[-1]) 

89 convergence = f_rel * np.abs(blend.loss[-1]) 

90 return deltaLoss < convergence 

91 

92 

93def _computePsfImage(psfModels, position, filters): 

94 """Get a multiband PSF image 

95 

96 The PSF Kernel Image is computed for each band 

97 and combined into a (filter, y, x) array. 

98 

99 Parameters 

100 ---------- 

101 psfList : `list` of `lsst.afw.detection.Psf` 

102 The list of PSFs in each band. 

103 position : `Point2D` or `tuple` 

104 Coordinates to evaluate the PSF. 

105 Returns 

106 ------- 

107 psfs: `np.ndarray` 

108 The multiband PSF image. 

109 """ 

110 psfs = [] 

111 # Make the coordinates into a Point2D (if necessary) 

112 if not isinstance(position, Point2D): 

113 position = Point2D(position[0], position[1]) 

114 

115 for bidx, psfModel in enumerate(psfModels): 

116 try: 

117 psf = psfModel.computeKernelImage(position) 

118 psfs.append(psf) 

119 except InvalidParameterError: 

120 # This band failed to compute the PSF due to incomplete data 

121 # at that location. This is unlikely to be a problem for Rubin, 

122 # however the edges of some HSC COSMOS fields contain incomplete 

123 # data in some bands, so we track this error to distinguish it 

124 # from unknown errors. 

125 msg = "Failed to compute PSF at {} in band {}" 

126 raise IncompleteDataError(msg.format(position, filters[bidx])) from None 

127 

128 left = np.min([psf.getBBox().getMinX() for psf in psfs]) 

129 bottom = np.min([psf.getBBox().getMinY() for psf in psfs]) 

130 right = np.max([psf.getBBox().getMaxX() for psf in psfs]) 

131 top = np.max([psf.getBBox().getMaxY() for psf in psfs]) 

132 bbox = Box2I(Point2I(left, bottom), Point2I(right, top)) 

133 psfs = np.array([afwImage.utils.projectImage(psf, bbox).array for psf in psfs]) 

134 return psfs 

135 

136 

137def getFootprintMask(footprint, mExposure): 

138 """Mask pixels outside the footprint 

139 

140 Parameters 

141 ---------- 

142 mExposure : `lsst.image.MultibandExposure` 

143 - The multiband exposure containing the image, 

144 mask, and variance data 

145 footprint : `lsst.detection.Footprint` 

146 - The footprint of the parent to deblend 

147 

148 Returns 

149 ------- 

150 footprintMask : array 

151 Boolean array with pixels not in the footprint set to one. 

152 """ 

153 bbox = footprint.getBBox() 

154 fpMask = afwImage.Mask(bbox) 

155 footprint.spans.setMask(fpMask, 1) 

156 fpMask = ~fpMask.getArray().astype(bool) 

157 return fpMask 

158 

159 

160def isPseudoSource(source, pseudoColumns): 

161 """Check if a source is a pseudo source. 

162 

163 This is mostly for skipping sky objects, 

164 but any other column can also be added to disable 

165 deblending on a parent or individual source when 

166 set to `True`. 

167 

168 Parameters 

169 ---------- 

170 source : `lsst.afw.table.source.source.SourceRecord` 

171 The source to check for the pseudo bit. 

172 pseudoColumns : `list` of `str` 

173 A list of columns to check for pseudo sources. 

174 """ 

175 isPseudo = False 

176 for col in pseudoColumns: 

177 try: 

178 isPseudo |= source[col] 

179 except KeyError: 

180 pass 

181 return isPseudo 

182 

183 

184def deblend(mExposure, footprint, config, spectrumInit): 

185 """Deblend a parent footprint 

186 

187 Parameters 

188 ---------- 

189 mExposure : `lsst.image.MultibandExposure` 

190 The multiband exposure containing the image, 

191 mask, and variance data. 

192 footprint : `lsst.detection.Footprint` 

193 The footprint of the parent to deblend. 

194 config : `ScarletDeblendConfig` 

195 Configuration of the deblending task. 

196 spectrumInit : `bool` 

197 Whether or not to initialize the spectrum. 

198 

199 Returns 

200 ------- 

201 blendData : `lsst.meas.extensions.scarlet.io.ScarletBlendData` 

202 The persistable representation of a `scarlet.Blend`. 

203 skipped : `list` of `int` 

204 The indices of any children that failed to initialize 

205 and were skipped. 

206 spectrumInit : `bool` 

207 Whether or not all of the sources were initialized by jointly 

208 fitting their SED's. This provides a better initialization 

209 but created memory issues when a blend is too large or 

210 contains too many sources. 

211 """ 

212 # Extract coordinates from each MultiColorPeak 

213 bbox = footprint.getBBox() 

214 

215 # Create the data array from the masked images 

216 images = mExposure.image[:, bbox].array 

217 

218 # Use the inverse variance as the weights 

219 if config.useWeights: 

220 weights = 1/mExposure.variance[:, bbox].array 

221 else: 

222 weights = np.ones_like(images) 

223 badPixels = mExposure.mask.getPlaneBitMask(config.badMask) 

224 mask = mExposure.mask[:, bbox].array & badPixels 

225 weights[mask > 0] = 0 

226 

227 # Mask out the pixels outside the footprint 

228 mask = getFootprintMask(footprint, mExposure) 

229 weights *= ~mask 

230 

231 psfCenter = footprint.getCentroid() 

232 psfModels = [exp.getPsf() for exp in mExposure] 

233 psfs = _computePsfImage(psfModels, psfCenter, mExposure.filters).astype(np.float32) 

234 psfs = ImagePSF(psfs) 

235 model_psf = GaussianPSF(sigma=(config.modelPsfSigma,)*len(mExposure.filters)) 

236 

237 frame = Frame(images.shape, psf=model_psf, channels=mExposure.filters) 

238 observation = Observation(images, psf=psfs, weights=weights, channels=mExposure.filters) 

239 if config.convolutionType == "fft": 

240 observation.match(frame) 

241 elif config.convolutionType == "real": 

242 renderer = ConvolutionRenderer(observation, frame, convolution_type="real") 

243 observation.match(frame, renderer=renderer) 

244 else: 

245 raise ValueError("Unrecognized convolution type {}".format(config.convolutionType)) 

246 

247 assert(config.sourceModel in ["single", "double", "compact", "fit"]) 

248 

249 # Set the appropriate number of components 

250 if config.sourceModel == "single": 

251 maxComponents = 1 

252 elif config.sourceModel == "double": 

253 maxComponents = 2 

254 elif config.sourceModel == "compact": 

255 maxComponents = 0 

256 elif config.sourceModel == "point": 

257 raise NotImplementedError("Point source photometry is currently not implemented") 

258 elif config.sourceModel == "fit": 

259 # It is likely in the future that there will be some heuristic 

260 # used to determine what type of model to use for each source, 

261 # but that has not yet been implemented (see DM-22551) 

262 raise NotImplementedError("sourceModel 'fit' has not been implemented yet") 

263 

264 # Convert the centers to pixel coordinates 

265 xmin = bbox.getMinX() 

266 ymin = bbox.getMinY() 

267 centers = [ 

268 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

269 for peak in footprint.peaks 

270 if not isPseudoSource(peak, config.pseudoColumns) 

271 ] 

272 

273 # Only deblend sources that can be initialized 

274 sources, skipped = init_all_sources( 

275 frame=frame, 

276 centers=centers, 

277 observations=observation, 

278 thresh=config.morphThresh, 

279 max_components=maxComponents, 

280 min_snr=config.minSNR, 

281 shifting=False, 

282 fallback=config.fallback, 

283 silent=config.catchFailures, 

284 set_spectra=spectrumInit, 

285 ) 

286 

287 # Attach the peak to all of the initialized sources 

288 srcIndex = 0 

289 for k, center in enumerate(centers): 

290 if k not in skipped: 

291 # This is just to make sure that there isn't a coding bug 

292 assert np.all(sources[srcIndex].center == center) 

293 # Store the record for the peak with the appropriate source 

294 sources[srcIndex].detectedPeak = footprint.peaks[k] 

295 srcIndex += 1 

296 

297 # Create the blend and attempt to optimize it 

298 blend = Blend(sources, observation) 

299 try: 

300 blend.fit(max_iter=config.maxIter, e_rel=config.relativeError) 

301 except ArithmeticError: 

302 # This occurs when a gradient update produces a NaN value 

303 # This is usually due to a source initialized with a 

304 # negative SED or no flux, often because the peak 

305 # is a noise fluctuation in one band and not a real source. 

306 iterations = len(blend.loss) 

307 failedSources = [] 

308 for k, src in enumerate(sources): 

309 if np.any(~np.isfinite(src.get_model())): 

310 failedSources.append(k) 

311 raise ScarletGradientError(iterations, failedSources) 

312 

313 # Store the location of the PSF center for storage 

314 blend.psfCenter = (psfCenter.x, psfCenter.y) 

315 

316 return blend, skipped 

317 

318 

319def deblend_lite(mExposure, modelPsf, footprint, config, spectrumInit, wavelets=None): 

320 """Deblend a parent footprint 

321 

322 Parameters 

323 ---------- 

324 mExposure : `lsst.image.MultibandExposure` 

325 - The multiband exposure containing the image, 

326 mask, and variance data 

327 footprint : `lsst.detection.Footprint` 

328 - The footprint of the parent to deblend 

329 config : `ScarletDeblendConfig` 

330 - Configuration of the deblending task 

331 """ 

332 # Extract coordinates from each MultiColorPeak 

333 bbox = footprint.getBBox() 

334 

335 # Create the data array from the masked images 

336 images = mExposure.image[:, bbox].array 

337 variance = mExposure.variance[:, bbox].array 

338 

339 # Use the inverse variance as the weights 

340 if config.useWeights: 

341 weights = 1/mExposure.variance[:, bbox].array 

342 else: 

343 weights = np.ones_like(images) 

344 badPixels = mExposure.mask.getPlaneBitMask(config.badMask) 

345 mask = mExposure.mask[:, bbox].array & badPixels 

346 weights[mask > 0] = 0 

347 

348 # Mask out the pixels outside the footprint 

349 mask = getFootprintMask(footprint, mExposure) 

350 weights *= ~mask 

351 

352 psfCenter = footprint.getCentroid() 

353 psfModels = [exp.getPsf() for exp in mExposure] 

354 psfs = _computePsfImage(psfModels, psfCenter, mExposure.filters).astype(np.float32) 

355 

356 observation = lite.LiteObservation( 

357 images=images, 

358 variance=variance, 

359 weights=weights, 

360 psfs=psfs, 

361 model_psf=modelPsf[None, :, :], 

362 convolution_mode=config.convolutionType, 

363 ) 

364 

365 # Convert the centers to pixel coordinates 

366 xmin = bbox.getMinX() 

367 ymin = bbox.getMinY() 

368 centers = [ 

369 np.array([peak.getIy() - ymin, peak.getIx() - xmin], dtype=int) 

370 for peak in footprint.peaks 

371 if not isPseudoSource(peak, config.pseudoColumns) 

372 ] 

373 

374 # Initialize the sources 

375 if config.morphImage == "chi2": 

376 sources = lite.init_all_sources_main( 

377 observation, 

378 centers, 

379 min_snr=config.minSNR, 

380 thresh=config.morphThresh, 

381 ) 

382 elif config.morphImage == "wavelet": 

383 _bbox = bboxToScarletBox(len(mExposure.filters), bbox, bbox.getMin()) 

384 _wavelets = wavelets[(slice(None), *_bbox[1:].slices)] 

385 sources = lite.init_all_sources_wavelets( 

386 observation, 

387 centers, 

388 use_psf=False, 

389 wavelets=_wavelets, 

390 min_snr=config.minSNR, 

391 ) 

392 else: 

393 raise ValueError("morphImage must be either 'chi2' or 'wavelet'.") 

394 

395 # Set the optimizer 

396 if config.optimizer == "adaprox": 

397 parameterization = partial( 

398 lite.init_adaprox_component, 

399 bg_thresh=config.backgroundThresh, 

400 max_prox_iter=config.maxProxIter, 

401 ) 

402 elif config.optimizer == "fista": 

403 parameterization = partial( 

404 lite.init_fista_component, 

405 bg_thresh=config.backgroundThresh, 

406 ) 

407 else: 

408 raise ValueError("Unrecognized optimizer. Must be either 'adaprox' or 'fista'.") 

409 sources = lite.parameterize_sources(sources, observation, parameterization) 

410 

411 # Attach the peak to all of the initialized sources 

412 for k, center in enumerate(centers): 

413 # This is just to make sure that there isn't a coding bug 

414 if len(sources[k].components) > 0 and np.any(sources[k].center != center): 

415 raise ValueError("Misaligned center, expected {center} but got {sources[k].center}") 

416 # Store the record for the peak with the appropriate source 

417 sources[k].detectedPeak = footprint.peaks[k] 

418 

419 blend = lite.LiteBlend(sources, observation) 

420 

421 # Initialize each source with its best fit spectrum 

422 if spectrumInit: 

423 blend.fit_spectra() 

424 

425 # Set the sources that could not be initialized and were skipped 

426 skipped = [src for src in sources if src.is_null] 

427 

428 blend.fit( 

429 max_iter=config.maxIter, 

430 e_rel=config.relativeError, 

431 min_iter=config.minIter, 

432 reweight=False, 

433 ) 

434 

435 # Store the location of the PSF center for storage 

436 blend.psfCenter = (psfCenter.x, psfCenter.y) 

437 

438 return blend, skipped 

439 

440 

441@dataclass 

442class DeblenderMetrics: 

443 """Metrics and measurements made on single sources. 

444 

445 Store deblender metrics to be added as attributes to a scarlet source 

446 before it is converted into a `SourceRecord`. 

447 When DM-34414 is finished this class will be eliminated and the metrics 

448 will be added to the schema using a pipeline task that calculates them 

449 from the stored deconvolved models. 

450 

451 All of the parameters are one dimensional numpy arrays, 

452 with an element for each band in the observed images. 

453 

454 `maxOverlap` is useful as a metric for determining how blended a source 

455 is because if it only overlaps with other sources at or below 

456 the noise level, it is likely to be a mostly isolated source 

457 in the deconvolved model frame. 

458 

459 `fluxOverlapFraction` is potentially more useful than the canonical 

460 "blendedness" (or purity) metric because it accounts for potential 

461 biases created during deblending by not weighting the overlapping 

462 flux with the flux of this sources model. 

463 

464 Attributes 

465 ---------- 

466 maxOverlap : `numpy.ndarray` 

467 The maximum overlap that the source has with its neighbors in 

468 a single pixel. 

469 fluxOverlap : `numpy.ndarray` 

470 The total flux from neighbors overlapping with the current source. 

471 fluxOverlapFraction : `numpy.ndarray` 

472 The fraction of `flux from neighbors/source flux` for a 

473 given source within the source's footprint. 

474 blendedness : `numpy.ndarray` 

475 The metric for determining how blended a source is using the 

476 Bosch et al. 2018 metric for "blendedness." Note that some 

477 surveys use the term "purity," which is `1-blendedness`. 

478 """ 

479 maxOverlap: np.array 

480 fluxOverlap: np.array 

481 fluxOverlapFraction: np.array 

482 blendedness: np.array 

483 

484 

485def setDeblenderMetrics(blend): 

486 """Set metrics that can be used to evalute the deblender accuracy 

487 

488 This function calculates the `DeblenderMetrics` for each source in the 

489 blend, and assigns it to that sources `metrics` property in place. 

490 

491 Parameters 

492 ---------- 

493 blend : `scarlet.lite.Blend` 

494 The blend containing the sources to measure. 

495 """ 

496 # Store the full model of the scene for comparison 

497 blendModel = blend.get_model() 

498 for k, src in enumerate(blend.sources): 

499 # Extract the source model in the full bounding box 

500 model = src.get_model(bbox=blend.bbox) 

501 # The footprint is the 2D array of non-zero pixels in each band 

502 footprint = np.bitwise_or.reduce(model > 0, axis=0) 

503 # Calculate the metrics. 

504 # See `DeblenderMetrics` for a description of each metric. 

505 neighborOverlap = (blendModel-model) * footprint[None, :, :] 

506 maxOverlap = np.max(neighborOverlap, axis=(1, 2)) 

507 fluxOverlap = np.sum(neighborOverlap, axis=(1, 2)) 

508 fluxModel = np.sum(model, axis=(1, 2)) 

509 fluxOverlapFraction = np.zeros((len(model), ), dtype=float) 

510 isFinite = fluxModel > 0 

511 fluxOverlapFraction[isFinite] = fluxOverlap[isFinite]/fluxModel[isFinite] 

512 blendedness = 1 - np.sum(model*model, axis=(1, 2))/np.sum(blendModel*model, axis=(1, 2)) 

513 src.metrics = DeblenderMetrics(maxOverlap, fluxOverlap, fluxOverlapFraction, blendedness) 

514 

515 

516class ScarletDeblendConfig(pexConfig.Config): 

517 """MultibandDeblendConfig 

518 

519 Configuration for the multiband deblender. 

520 The parameters are organized by the parameter types, which are 

521 - Stopping Criteria: Used to determine if the fit has converged 

522 - Position Fitting Criteria: Used to fit the positions of the peaks 

523 - Constraints: Used to apply constraints to the peaks and their components 

524 - Other: Parameters that don't fit into the above categories 

525 """ 

526 # Stopping Criteria 

527 minIter = pexConfig.Field(dtype=int, default=1, 

528 doc="Minimum number of iterations before the optimizer is allowed to stop.") 

529 maxIter = pexConfig.Field(dtype=int, default=300, 

530 doc=("Maximum number of iterations to deblend a single parent")) 

531 relativeError = pexConfig.Field(dtype=float, default=1e-2, 

532 doc=("Change in the loss function between iterations to exit fitter. " 

533 "Typically this is `1e-2` if measurements will be made on the " 

534 "flux re-distributed models and `1e-4` when making measurements " 

535 "on the models themselves.")) 

536 

537 # Constraints 

538 morphThresh = pexConfig.Field(dtype=float, default=1, 

539 doc="Fraction of background RMS a pixel must have" 

540 "to be included in the initial morphology") 

541 # Lite Parameters 

542 # All of these parameters (except version) are only valid if version='lite' 

543 version = pexConfig.ChoiceField( 

544 dtype=str, 

545 default="lite", 

546 allowed={ 

547 "scarlet": "main scarlet version (likely to be deprecated soon)", 

548 "lite": "Optimized version of scarlet for survey data from a single instrument", 

549 }, 

550 doc="The version of scarlet to use.", 

551 ) 

552 optimizer = pexConfig.ChoiceField( 

553 dtype=str, 

554 default="adaprox", 

555 allowed={ 

556 "adaprox": "Proximal ADAM optimization", 

557 "fista": "Accelerated proximal gradient method", 

558 }, 

559 doc="The optimizer to use for fitting parameters and is only used when version='lite'", 

560 ) 

561 morphImage = pexConfig.ChoiceField( 

562 dtype=str, 

563 default="chi2", 

564 allowed={ 

565 "chi2": "Initialize sources on a chi^2 image made from all available bands", 

566 "wavelet": "Initialize sources using a wavelet decomposition of the chi^2 image", 

567 }, 

568 doc="The type of image to use for initializing the morphology. " 

569 "Must be either 'chi2' or 'wavelet'. " 

570 ) 

571 backgroundThresh = pexConfig.Field( 

572 dtype=float, 

573 default=0.25, 

574 doc="Fraction of background to use for a sparsity threshold. " 

575 "This prevents sources from growing unrealistically outside " 

576 "the parent footprint while still modeling flux correctly " 

577 "for bright sources." 

578 ) 

579 maxProxIter = pexConfig.Field( 

580 dtype=int, 

581 default=1, 

582 doc="Maximum number of proximal operator iterations inside of each " 

583 "iteration of the optimizer. " 

584 "This config field is only used if version='lite' and optimizer='adaprox'." 

585 ) 

586 waveletScales = pexConfig.Field( 

587 dtype=int, 

588 default=5, 

589 doc="Number of wavelet scales to use for wavelet initialization. " 

590 "This field is only used when `version`='lite' and `morphImage`='wavelet'." 

591 ) 

592 

593 # Other scarlet paremeters 

594 useWeights = pexConfig.Field( 

595 dtype=bool, default=True, 

596 doc=("Whether or not use use inverse variance weighting." 

597 "If `useWeights` is `False` then flat weights are used")) 

598 modelPsfSize = pexConfig.Field( 

599 dtype=int, default=11, 

600 doc="Model PSF side length in pixels") 

601 modelPsfSigma = pexConfig.Field( 

602 dtype=float, default=0.8, 

603 doc="Define sigma for the model frame PSF") 

604 minSNR = pexConfig.Field( 

605 dtype=float, default=50, 

606 doc="Minimum Signal to noise to accept the source." 

607 "Sources with lower flux will be initialized with the PSF but updated " 

608 "like an ordinary ExtendedSource (known in scarlet as a `CompactSource`).") 

609 saveTemplates = pexConfig.Field( 

610 dtype=bool, default=True, 

611 doc="Whether or not to save the SEDs and templates") 

612 processSingles = pexConfig.Field( 

613 dtype=bool, default=True, 

614 doc="Whether or not to process isolated sources in the deblender") 

615 convolutionType = pexConfig.Field( 

616 dtype=str, default="fft", 

617 doc="Type of convolution to render the model to the observations.\n" 

618 "- 'fft': perform convolutions in Fourier space\n" 

619 "- 'real': peform convolutions in real space.") 

620 sourceModel = pexConfig.Field( 

621 dtype=str, default="double", 

622 doc=("How to determine which model to use for sources, from\n" 

623 "- 'single': use a single component for all sources\n" 

624 "- 'double': use a bulge disk model for all sources\n" 

625 "- 'compact': use a single component model, initialzed with a point source morphology, " 

626 " for all sources\n" 

627 "- 'point': use a point-source model for all sources\n" 

628 "- 'fit: use a PSF fitting model to determine the number of components (not yet implemented)"), 

629 deprecated="This field will be deprecated when the default for `version` is changed to `lite`.", 

630 ) 

631 setSpectra = pexConfig.Field( 

632 dtype=bool, default=True, 

633 doc="Whether or not to solve for the best-fit spectra during initialization. " 

634 "This makes initialization slightly longer, as it requires a convolution " 

635 "to set the optimal spectra, but results in a much better initial log-likelihood " 

636 "and reduced total runtime, with convergence in fewer iterations." 

637 "This option is only used when " 

638 "peaks*area < `maxSpectrumCutoff` will use the improved initialization.") 

639 

640 # Mask-plane restrictions 

641 badMask = pexConfig.ListField( 

642 dtype=str, default=["BAD", "CR", "NO_DATA", "SAT", "SUSPECT", "EDGE"], 

643 doc="Whether or not to process isolated sources in the deblender") 

644 statsMask = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

645 doc="Mask planes to ignore when performing statistics") 

646 maskLimits = pexConfig.DictField( 

647 keytype=str, 

648 itemtype=float, 

649 default={}, 

650 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

651 "Sources violating this limit will not be deblended. " 

652 "If the fraction is `0` then the limit is a single pixel."), 

653 ) 

654 

655 # Size restrictions 

656 maxNumberOfPeaks = pexConfig.Field( 

657 dtype=int, default=200, 

658 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

659 " (<= 0: unlimited)")) 

660 maxFootprintArea = pexConfig.Field( 

661 dtype=int, default=100_000, 

662 doc=("Maximum area for footprints before they are ignored as large; " 

663 "non-positive means no threshold applied")) 

664 maxAreaTimesPeaks = pexConfig.Field( 

665 dtype=int, default=10_000_000, 

666 doc=("Maximum rectangular footprint area * nPeaks in the footprint. " 

667 "This was introduced in DM-33690 to prevent fields that are crowded or have a " 

668 "LSB galaxy that causes memory intensive initialization in scarlet from dominating " 

669 "the overall runtime and/or causing the task to run out of memory. " 

670 "(<= 0: unlimited)") 

671 ) 

672 maxFootprintSize = pexConfig.Field( 

673 dtype=int, default=0, 

674 doc=("Maximum linear dimension for footprints before they are ignored " 

675 "as large; non-positive means no threshold applied")) 

676 minFootprintAxisRatio = pexConfig.Field( 

677 dtype=float, default=0.0, 

678 doc=("Minimum axis ratio for footprints before they are ignored " 

679 "as large; non-positive means no threshold applied")) 

680 maxSpectrumCutoff = pexConfig.Field( 

681 dtype=int, default=1_000_000, 

682 doc=("Maximum number of pixels * number of sources in a blend. " 

683 "This is different than `maxFootprintArea` because this isn't " 

684 "the footprint area but the area of the bounding box that " 

685 "contains the footprint, and is also multiplied by the number of" 

686 "sources in the footprint. This prevents large skinny blends with " 

687 "a high density of sources from running out of memory. " 

688 "If `maxSpectrumCutoff == -1` then there is no cutoff.") 

689 ) 

690 # Failure modes 

691 fallback = pexConfig.Field( 

692 dtype=bool, default=True, 

693 doc="Whether or not to fallback to a smaller number of components if a source does not initialize" 

694 ) 

695 notDeblendedMask = pexConfig.Field( 

696 dtype=str, default="NOT_DEBLENDED", optional=True, 

697 doc="Mask name for footprints not deblended, or None") 

698 catchFailures = pexConfig.Field( 

699 dtype=bool, default=True, 

700 doc=("If True, catch exceptions thrown by the deblender, log them, " 

701 "and set a flag on the parent, instead of letting them propagate up")) 

702 

703 # Other options 

704 columnInheritance = pexConfig.DictField( 

705 keytype=str, itemtype=str, default={ 

706 "deblend_nChild": "deblend_parentNChild", 

707 "deblend_nPeaks": "deblend_parentNPeaks", 

708 "deblend_spectrumInitFlag": "deblend_spectrumInitFlag", 

709 "deblend_blendConvergenceFailedFlag": "deblend_blendConvergenceFailedFlag", 

710 }, 

711 doc="Columns to pass from the parent to the child. " 

712 "The key is the name of the column for the parent record, " 

713 "the value is the name of the column to use for the child." 

714 ) 

715 pseudoColumns = pexConfig.ListField( 

716 dtype=str, default=['merge_peak_sky', 'sky_source'], 

717 doc="Names of flags which should never be deblended." 

718 ) 

719 

720 # Logging option(s) 

721 loggingInterval = pexConfig.Field( 

722 dtype=int, default=600, 

723 doc="Interval (in seconds) to log messages (at VERBOSE level) while deblending sources.", 

724 deprecated="This field is no longer used and will be removed in v25.", 

725 ) 

726 # Testing options 

727 # Some obs packages and ci packages run the full pipeline on a small 

728 # subset of data to test that the pipeline is functioning properly. 

729 # This is not meant as scientific validation, so it can be useful 

730 # to only run on a small subset of the data that is large enough to 

731 # test the desired pipeline features but not so long that the deblender 

732 # is the tall pole in terms of execution times. 

733 useCiLimits = pexConfig.Field( 

734 dtype=bool, default=False, 

735 doc="Limit the number of sources deblended for CI to prevent long build times") 

736 ciDeblendChildRange = pexConfig.ListField( 

737 dtype=int, default=[5, 10], 

738 doc="Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated." 

739 "If `useCiLimits==False` then this parameter is ignored.") 

740 ciNumParentsToDeblend = pexConfig.Field( 

741 dtype=int, default=10, 

742 doc="Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count " 

743 "within `ciDebledChildRange`. " 

744 "If `useCiLimits==False` then this parameter is ignored.") 

745 

746 

747class ScarletDeblendTask(pipeBase.Task): 

748 """ScarletDeblendTask 

749 

750 Split blended sources into individual sources. 

751 

752 This task has no return value; it only modifies the SourceCatalog in-place. 

753 """ 

754 ConfigClass = ScarletDeblendConfig 

755 _DefaultName = "scarletDeblend" 

756 

757 def __init__(self, schema, peakSchema=None, **kwargs): 

758 """Create the task, adding necessary fields to the given schema. 

759 

760 Parameters 

761 ---------- 

762 schema : `lsst.afw.table.schema.schema.Schema` 

763 Schema object for measurement fields; will be modified in-place. 

764 peakSchema : `lsst.afw.table.schema.schema.Schema` 

765 Schema of Footprint Peaks that will be passed to the deblender. 

766 Any fields beyond the PeakTable minimal schema will be transferred 

767 to the main source Schema. If None, no fields will be transferred 

768 from the Peaks. 

769 filters : list of str 

770 Names of the filters used for the eposures. This is needed to store 

771 the SED as a field 

772 **kwargs 

773 Passed to Task.__init__. 

774 """ 

775 pipeBase.Task.__init__(self, **kwargs) 

776 

777 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

778 if peakSchema is None: 

779 # In this case, the peakSchemaMapper will transfer nothing, but 

780 # we'll still have one 

781 # to simplify downstream code 

782 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

783 else: 

784 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

785 for item in peakSchema: 

786 if item.key not in peakMinimalSchema: 

787 self.peakSchemaMapper.addMapping(item.key, item.field) 

788 # Because SchemaMapper makes a copy of the output schema 

789 # you give its ctor, it isn't updating this Schema in 

790 # place. That's probably a design flaw, but in the 

791 # meantime, we'll keep that schema in sync with the 

792 # peakSchemaMapper.getOutputSchema() manually, by adding 

793 # the same fields to both. 

794 schema.addField(item.field) 

795 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

796 self._addSchemaKeys(schema) 

797 self.schema = schema 

798 self.toCopyFromParent = [item.key for item in self.schema 

799 if item.field.getName().startswith("merge_footprint")] 

800 

801 def _addSchemaKeys(self, schema): 

802 """Add deblender specific keys to the schema 

803 """ 

804 # Parent (blend) fields 

805 self.runtimeKey = schema.addField('deblend_runtime', type=np.float32, doc='runtime in ms') 

806 self.iterKey = schema.addField('deblend_iterations', type=np.int32, doc='iterations to converge') 

807 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

808 doc='Number of children this object has (defaults to 0)') 

809 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32, 

810 doc="Number of initial peaks in the blend. " 

811 "This includes peaks that may have been culled " 

812 "during deblending or failed to deblend") 

813 # Skipped flags 

814 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

815 doc="Deblender skipped this source") 

816 self.isolatedParentKey = schema.addField('deblend_isolatedParent', type='Flag', 

817 doc='The source has only a single peak ' 

818 'and was not deblended') 

819 self.pseudoKey = schema.addField('deblend_isPseudo', type='Flag', 

820 doc='The source is identified as a "pseudo" source and ' 

821 'was not deblended') 

822 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

823 doc='Source had too many peaks; ' 

824 'only the brightest were included') 

825 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

826 doc='Parent footprint covered too many pixels') 

827 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

828 doc='Parent footprint had too many masked pixels') 

829 # Convergence flags 

830 self.sedNotConvergedKey = schema.addField('deblend_sedConvergenceFailed', type='Flag', 

831 doc='scarlet sed optimization did not converge before' 

832 'config.maxIter') 

833 self.morphNotConvergedKey = schema.addField('deblend_morphConvergenceFailed', type='Flag', 

834 doc='scarlet morph optimization did not converge before' 

835 'config.maxIter') 

836 self.blendConvergenceFailedFlagKey = schema.addField('deblend_blendConvergenceFailedFlag', 

837 type='Flag', 

838 doc='at least one source in the blend' 

839 'failed to converge') 

840 # Error flags 

841 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

842 doc="Deblending failed on source") 

843 self.deblendErrorKey = schema.addField('deblend_error', type="String", size=25, 

844 doc='Name of error if the blend failed') 

845 # Deblended source fields 

846 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center", 

847 doc="Center used to apply constraints in scarlet", 

848 unit="pixel") 

849 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32, 

850 doc="ID of the peak in the parent footprint. " 

851 "This is not unique, but the combination of 'parent'" 

852 "and 'peakId' should be for all child sources. " 

853 "Top level blends with no parents have 'peakId=0'") 

854 self.modelCenterFlux = schema.addField('deblend_peak_instFlux', type=float, units='count', 

855 doc="The instFlux at the peak position of deblended mode") 

856 self.modelTypeKey = schema.addField("deblend_modelType", type="String", size=25, 

857 doc="The type of model used, for example " 

858 "MultiExtendedSource, SingleExtendedSource, PointSource") 

859 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32, 

860 doc="deblend_nPeaks from this records parent.") 

861 self.parentNChildKey = schema.addField("deblend_parentNChild", type=np.int32, 

862 doc="deblend_nChild from this records parent.") 

863 self.scarletFluxKey = schema.addField("deblend_scarletFlux", type=np.float32, 

864 doc="Flux measurement from scarlet") 

865 self.scarletLogLKey = schema.addField("deblend_logL", type=np.float32, 

866 doc="Final logL, used to identify regressions in scarlet.") 

867 self.edgePixelsKey = schema.addField('deblend_edgePixels', type='Flag', 

868 doc='Source had flux on the edge of the parent footprint') 

869 self.scarletSpectrumInitKey = schema.addField("deblend_spectrumInitFlag", type='Flag', 

870 doc="True when scarlet initializes sources " 

871 "in the blend with a more accurate spectrum. " 

872 "The algorithm uses a lot of memory, " 

873 "so large dense blends will use " 

874 "a less accurate initialization.") 

875 self.nComponentsKey = schema.addField("deblend_nComponents", type=np.int32, 

876 doc="Number of components in a ScarletLiteSource. " 

877 "If `config.version != 'lite'`then " 

878 "this column is set to zero.") 

879 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

880 doc='Deblender thought this source looked like a PSF') 

881 # Blendedness/classification metrics 

882 self.maxOverlapKey = schema.addField("deblend_maxOverlap", type=np.float32, 

883 doc="Maximum overlap with all of the other neighbors flux " 

884 "combined." 

885 "This is useful as a metric for determining how blended a " 

886 "source is because if it only overlaps with other sources " 

887 "at or below the noise level, it is likely to be a mostly " 

888 "isolated source in the deconvolved model frame.") 

889 self.fluxOverlapKey = schema.addField("deblend_fluxOverlap", type=np.float32, 

890 doc="This is the total flux from neighboring objects that " 

891 "overlaps with this source.") 

892 self.fluxOverlapFractionKey = schema.addField("deblend_fluxOverlapFraction", type=np.float32, 

893 doc="This is the fraction of " 

894 "`flux from neighbors/source flux` " 

895 "for a given source within the source's" 

896 "footprint.") 

897 self.blendednessKey = schema.addField("deblend_blendedness", type=np.float32, 

898 doc="The Bosch et al. 2018 metric for 'blendedness.' ") 

899 

900 @timeMethod 

901 def run(self, mExposure, mergedSources): 

902 """Get the psf from each exposure and then run deblend(). 

903 

904 Parameters 

905 ---------- 

906 mExposure : `MultibandExposure` 

907 The exposures should be co-added images of the same 

908 shape and region of the sky. 

909 mergedSources : `SourceCatalog` 

910 The merged `SourceCatalog` that contains parent footprints 

911 to (potentially) deblend. 

912 

913 Returns 

914 ------- 

915 templateCatalogs: dict 

916 Keys are the names of the filters and the values are 

917 `lsst.afw.table.source.source.SourceCatalog`'s. 

918 These are catalogs with heavy footprints that are the templates 

919 created by the multiband templates. 

920 """ 

921 return self.deblend(mExposure, mergedSources) 

922 

923 @timeMethod 

924 def deblend(self, mExposure, catalog): 

925 """Deblend a data cube of multiband images 

926 

927 Parameters 

928 ---------- 

929 mExposure : `MultibandExposure` 

930 The exposures should be co-added images of the same 

931 shape and region of the sky. 

932 catalog : `SourceCatalog` 

933 The merged `SourceCatalog` that contains parent footprints 

934 to (potentially) deblend. The new deblended sources are 

935 appended to this catalog in place. 

936 

937 Returns 

938 ------- 

939 catalogs : `dict` or `None` 

940 Keys are the names of the filters and the values are 

941 `lsst.afw.table.source.source.SourceCatalog`'s. 

942 These are catalogs with heavy footprints that are the templates 

943 created by the multiband templates. 

944 """ 

945 import time 

946 

947 # Cull footprints if required by ci 

948 if self.config.useCiLimits: 

949 self.log.info("Using CI catalog limits, the original number of sources to deblend was %d.", 

950 len(catalog)) 

951 # Select parents with a number of children in the range 

952 # config.ciDeblendChildRange 

953 minChildren, maxChildren = self.config.ciDeblendChildRange 

954 nPeaks = np.array([len(src.getFootprint().peaks) for src in catalog]) 

955 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0] 

956 if len(childrenInRange) < self.config.ciNumParentsToDeblend: 

957 raise ValueError("Fewer than ciNumParentsToDeblend children were contained in the range " 

958 "indicated by ciDeblendChildRange. Adjust this range to include more " 

959 "parents.") 

960 # Keep all of the isolated parents and the first 

961 # `ciNumParentsToDeblend` children 

962 parents = nPeaks == 1 

963 children = np.zeros((len(catalog),), dtype=bool) 

964 children[childrenInRange[:self.config.ciNumParentsToDeblend]] = True 

965 catalog = catalog[parents | children] 

966 # We need to update the IdFactory, otherwise the the source ids 

967 # will not be sequential 

968 idFactory = catalog.getIdFactory() 

969 maxId = np.max(catalog["id"]) 

970 idFactory.notify(maxId) 

971 

972 filters = mExposure.filters 

973 self.log.info("Deblending %d sources in %d exposure bands", len(catalog), len(mExposure)) 

974 periodicLog = PeriodicLogger(self.log) 

975 

976 # Create a set of wavelet coefficients if using wavelet initialization 

977 if self.config.version == "lite" and self.config.morphImage == "wavelet": 

978 images = mExposure.image.array 

979 variance = mExposure.variance.array 

980 wavelets = get_detect_wavelets(images, variance, scales=self.config.waveletScales) 

981 else: 

982 wavelets = None 

983 

984 # Add the NOT_DEBLENDED mask to the mask plane in each band 

985 if self.config.notDeblendedMask: 

986 for mask in mExposure.mask: 

987 mask.addMaskPlane(self.config.notDeblendedMask) 

988 

989 # Initialize the persistable data model 

990 modelPsf = lite.integrated_circular_gaussian(sigma=self.config.modelPsfSigma) 

991 dataModel = ScarletModelData(filters, modelPsf) 

992 

993 nParents = len(catalog) 

994 nDeblendedParents = 0 

995 skippedParents = [] 

996 for parentIndex in range(nParents): 

997 parent = catalog[parentIndex] 

998 foot = parent.getFootprint() 

999 bbox = foot.getBBox() 

1000 peaks = foot.getPeaks() 

1001 

1002 # Since we use the first peak for the parent object, we should 

1003 # propagate its flags to the parent source. 

1004 parent.assign(peaks[0], self.peakSchemaMapper) 

1005 

1006 # Block of conditions for skipping a parent with multiple children 

1007 if (skipArgs := self._checkSkipped(parent, mExposure)) is not None: 

1008 self._skipParent(parent, *skipArgs) 

1009 skippedParents.append(parentIndex) 

1010 continue 

1011 

1012 nDeblendedParents += 1 

1013 self.log.trace("Parent %d: deblending %d peaks", parent.getId(), len(peaks)) 

1014 # Run the deblender 

1015 blendError = None 

1016 

1017 # Choose whether or not to use improved spectral initialization. 

1018 # This significantly cuts down on the number of iterations 

1019 # that the optimizer needs and usually results in a better 

1020 # fit. 

1021 # But using least squares on a very large blend causes memory 

1022 # issues, so it is not done for large blends 

1023 if self.config.setSpectra: 

1024 if self.config.maxSpectrumCutoff <= 0: 

1025 spectrumInit = True 

1026 else: 

1027 spectrumInit = len(foot.peaks) * bbox.getArea() < self.config.maxSpectrumCutoff 

1028 else: 

1029 spectrumInit = False 

1030 

1031 try: 

1032 t0 = time.monotonic() 

1033 # Build the parameter lists with the same ordering 

1034 if self.config.version == "scarlet": 

1035 blend, skipped = deblend(mExposure, foot, self.config, spectrumInit) 

1036 elif self.config.version == "lite": 

1037 blend, skipped = deblend_lite( 

1038 mExposure=mExposure, 

1039 modelPsf=modelPsf, 

1040 footprint=foot, 

1041 config=self.config, 

1042 spectrumInit=spectrumInit, 

1043 wavelets=wavelets, 

1044 ) 

1045 tf = time.monotonic() 

1046 runtime = (tf-t0)*1000 

1047 converged = _checkBlendConvergence(blend, self.config.relativeError) 

1048 # Store the number of components in the blend 

1049 if self.config.version == "lite": 

1050 nComponents = len(blend.components) 

1051 else: 

1052 nComponents = 0 

1053 nChild = len(blend.sources) 

1054 # Catch all errors and filter out the ones that we know about 

1055 except Exception as e: 

1056 blendError = type(e).__name__ 

1057 if isinstance(e, ScarletGradientError): 

1058 parent.set(self.iterKey, e.iterations) 

1059 elif not isinstance(e, IncompleteDataError): 

1060 blendError = "UnknownError" 

1061 if self.config.catchFailures: 

1062 # Make it easy to find UnknownErrors in the log file 

1063 self.log.warn("UnknownError") 

1064 import traceback 

1065 traceback.print_exc() 

1066 else: 

1067 raise 

1068 

1069 self._skipParent( 

1070 parent=parent, 

1071 skipKey=self.deblendFailedKey, 

1072 logMessage=f"Unable to deblend source {parent.getId}: {blendError}", 

1073 ) 

1074 parent.set(self.deblendErrorKey, blendError) 

1075 skippedParents.append(parentIndex) 

1076 continue 

1077 

1078 # Update the parent record with the deblending results 

1079 if self.config.version == "scarlet": 

1080 logL = -blend.loss[-1] + blend.observations[0].log_norm 

1081 elif self.config.version == "lite": 

1082 logL = blend.loss[-1] 

1083 self._updateParentRecord( 

1084 parent=parent, 

1085 nPeaks=len(peaks), 

1086 nChild=nChild, 

1087 nComponents=nComponents, 

1088 runtime=runtime, 

1089 iterations=len(blend.loss), 

1090 logL=logL, 

1091 spectrumInit=spectrumInit, 

1092 converged=converged, 

1093 ) 

1094 

1095 # Add each deblended source to the catalog 

1096 for k, scarletSource in enumerate(blend.sources): 

1097 # Skip any sources with no flux or that scarlet skipped because 

1098 # it could not initialize 

1099 if k in skipped or (self.config.version == "lite" and scarletSource.is_null): 

1100 # No need to propagate anything 

1101 continue 

1102 parent.set(self.deblendSkippedKey, False) 

1103 

1104 # Add all fields except the HeavyFootprint to the 

1105 # source record 

1106 sourceRecord = self._addChild( 

1107 parent=parent, 

1108 peak=scarletSource.detectedPeak, 

1109 catalog=catalog, 

1110 scarletSource=scarletSource, 

1111 ) 

1112 scarletSource.recordId = sourceRecord.getId() 

1113 scarletSource.peakId = scarletSource.detectedPeak.getId() 

1114 

1115 # Store the blend information so that it can be persisted 

1116 if self.config.version == "lite": 

1117 blendData = scarletLiteToData(blend, blend.psfCenter, bbox.getMin()) 

1118 else: 

1119 blendData = scarletToData(blend, blend.psfCenter, bbox.getMin()) 

1120 dataModel.blends[parent.getId()] = blendData 

1121 

1122 # Log a message if it has been a while since the last log. 

1123 periodicLog.log("Deblended %d parent sources out of %d", parentIndex + 1, nParents) 

1124 

1125 # Clear the cached values in scarlet to clear out memory 

1126 scarlet.cache.Cache._cache = {} 

1127 

1128 # Update the mExposure mask with the footprint of skipped parents 

1129 if self.config.notDeblendedMask: 

1130 for mask in mExposure.mask: 

1131 for parentIndex in skippedParents: 

1132 fp = catalog[parentIndex].getFootprint() 

1133 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask)) 

1134 

1135 self.log.info("Deblender results: of %d parent sources, %d were deblended, " 

1136 "creating %d children, for a total of %d sources", 

1137 nParents, nDeblendedParents, len(catalog)-nParents, len(catalog)) 

1138 return catalog, dataModel 

1139 

1140 def _isLargeFootprint(self, footprint): 

1141 """Returns whether a Footprint is large 

1142 

1143 'Large' is defined by thresholds on the area, size and axis ratio, 

1144 and total area of the bounding box multiplied by 

1145 the number of children. 

1146 These may be disabled independently by configuring them to be 

1147 non-positive. 

1148 """ 

1149 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

1150 return True 

1151 if self.config.maxFootprintSize > 0: 

1152 bbox = footprint.getBBox() 

1153 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

1154 return True 

1155 if self.config.minFootprintAxisRatio > 0: 

1156 axes = afwEll.Axes(footprint.getShape()) 

1157 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

1158 return True 

1159 if self.config.maxAreaTimesPeaks > 0: 

1160 if footprint.getBBox().getArea() * len(footprint.peaks) > self.config.maxAreaTimesPeaks: 

1161 return True 

1162 return False 

1163 

1164 def _isMasked(self, footprint, mExposure): 

1165 """Returns whether the footprint violates the mask limits 

1166 

1167 Parameters 

1168 ---------- 

1169 footprint : `lsst.afw.detection.Footprint` 

1170 The footprint to check for masked pixels 

1171 mMask : `lsst.afw.image.MaskX` 

1172 The mask plane to check for masked pixels in the `footprint`. 

1173 

1174 Returns 

1175 ------- 

1176 isMasked : `bool` 

1177 `True` if `self.config.maskPlaneLimits` is less than the 

1178 fraction of pixels for a given mask in 

1179 `self.config.maskLimits`. 

1180 """ 

1181 bbox = footprint.getBBox() 

1182 mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0) 

1183 size = float(footprint.getArea()) 

1184 for maskName, limit in self.config.maskLimits.items(): 

1185 maskVal = mExposure.mask.getPlaneBitMask(maskName) 

1186 _mask = afwImage.MaskX(mask & maskVal, xy0=bbox.getMin()) 

1187 # spanset of masked pixels 

1188 maskedSpan = footprint.spans.intersect(_mask, maskVal) 

1189 if (maskedSpan.getArea())/size > limit: 

1190 return True 

1191 return False 

1192 

1193 def _skipParent(self, parent, skipKey, logMessage): 

1194 """Update a parent record that is not being deblended. 

1195 

1196 This is a fairly trivial function but is implemented to ensure 

1197 that a skipped parent updates the appropriate columns 

1198 consistently, and always has a flag to mark the reason that 

1199 it is being skipped. 

1200 

1201 Parameters 

1202 ---------- 

1203 parent : `lsst.afw.table.source.source.SourceRecord` 

1204 The parent record to flag as skipped. 

1205 skipKey : `bool` 

1206 The name of the flag to mark the reason for skipping. 

1207 logMessage : `str` 

1208 The message to display in a log.trace when a source 

1209 is skipped. 

1210 """ 

1211 if logMessage is not None: 

1212 self.log.trace(logMessage) 

1213 self._updateParentRecord( 

1214 parent=parent, 

1215 nPeaks=len(parent.getFootprint().peaks), 

1216 nChild=0, 

1217 nComponents=0, 

1218 runtime=np.nan, 

1219 iterations=0, 

1220 logL=np.nan, 

1221 spectrumInit=False, 

1222 converged=False, 

1223 ) 

1224 

1225 # Mark the source as skipped by the deblender and 

1226 # flag the reason why. 

1227 parent.set(self.deblendSkippedKey, True) 

1228 parent.set(skipKey, True) 

1229 

1230 def _checkSkipped(self, parent, mExposure): 

1231 """Update a parent record that is not being deblended. 

1232 

1233 This is a fairly trivial function but is implemented to ensure 

1234 that a skipped parent updates the appropriate columns 

1235 consistently, and always has a flag to mark the reason that 

1236 it is being skipped. 

1237 

1238 Parameters 

1239 ---------- 

1240 parent : `lsst.afw.table.source.source.SourceRecord` 

1241 The parent record to flag as skipped. 

1242 mExposure : `MultibandExposure` 

1243 The exposures should be co-added images of the same 

1244 shape and region of the sky. 

1245 Returns 

1246 ------- 

1247 skip: `bool` 

1248 `True` if the deblender will skip the parent 

1249 """ 

1250 skipKey = None 

1251 skipMessage = None 

1252 footprint = parent.getFootprint() 

1253 if len(footprint.peaks) < 2 and not self.config.processSingles: 

1254 # Skip isolated sources unless processSingles is turned on. 

1255 # Note: this does not flag isolated sources as skipped or 

1256 # set the NOT_DEBLENDED mask in the exposure, 

1257 # since these aren't really any skipped blends. 

1258 skipKey = self.isolatedParentKey 

1259 elif isPseudoSource(parent, self.config.pseudoColumns): 

1260 # We also skip pseudo sources, like sky objects, which 

1261 # are intended to be skipped. 

1262 skipKey = self.pseudoKey 

1263 if self._isLargeFootprint(footprint): 

1264 # The footprint is above the maximum footprint size limit 

1265 skipKey = self.tooBigKey 

1266 skipMessage = f"Parent {parent.getId()}: skipping large footprint" 

1267 elif self._isMasked(footprint, mExposure): 

1268 # The footprint exceeds the maximum number of masked pixels 

1269 skipKey = self.maskedKey 

1270 skipMessage = f"Parent {parent.getId()}: skipping masked footprint" 

1271 elif self.config.maxNumberOfPeaks > 0 and len(footprint.peaks) > self.config.maxNumberOfPeaks: 

1272 # Unlike meas_deblender, in scarlet we skip the entire blend 

1273 # if the number of peaks exceeds max peaks, since neglecting 

1274 # to model any peaks often results in catastrophic failure 

1275 # of scarlet to generate models for the brighter sources. 

1276 skipKey = self.tooManyPeaksKey 

1277 skipMessage = f"Parent {parent.getId()}: skipping blend with too many peaks" 

1278 if skipKey is not None: 

1279 return (skipKey, skipMessage) 

1280 return None 

1281 

1282 def setSkipFlags(self, mExposure, catalog): 

1283 """Set the skip flags for all of the parent sources 

1284 

1285 This is mostly used for testing which parent sources will be deblended 

1286 and which will be skipped based on the current configuration options. 

1287 Skipped sources will have the appropriate flags set in place in the 

1288 catalog. 

1289 

1290 Parameters 

1291 ---------- 

1292 mExposure : `MultibandExposure` 

1293 The exposures should be co-added images of the same 

1294 shape and region of the sky. 

1295 catalog : `SourceCatalog` 

1296 The merged `SourceCatalog` that contains parent footprints 

1297 to (potentially) deblend. The new deblended sources are 

1298 appended to this catalog in place. 

1299 """ 

1300 for src in catalog: 

1301 if skipArgs := self._checkSkipped(src, mExposure) is not None: 

1302 self._skipParent(src, *skipArgs) 

1303 

1304 def _updateParentRecord(self, parent, nPeaks, nChild, nComponents, 

1305 runtime, iterations, logL, spectrumInit, converged): 

1306 """Update a parent record in all of the single band catalogs. 

1307 

1308 Ensure that all locations that update a parent record, 

1309 whether it is skipped or updated after deblending, 

1310 update all of the appropriate columns. 

1311 

1312 Parameters 

1313 ---------- 

1314 parent : `lsst.afw.table.source.source.SourceRecord` 

1315 The parent record to update. 

1316 nPeaks : `int` 

1317 Number of peaks in the parent footprint. 

1318 nChild : `int` 

1319 Number of children deblended from the parent. 

1320 This may differ from `nPeaks` if some of the peaks 

1321 were culled and have no deblended model. 

1322 nComponents : `int` 

1323 Total number of components in the parent. 

1324 This is usually different than the number of children, 

1325 since it is common for a single source to have multiple 

1326 components. 

1327 runtime : `float` 

1328 Total runtime for deblending. 

1329 iterations : `int` 

1330 Total number of iterations in scarlet before convergence. 

1331 logL : `float` 

1332 Final log likelihood of the blend. 

1333 spectrumInit : `bool` 

1334 True when scarlet used `set_spectra` to initialize all 

1335 sources with better initial intensities. 

1336 converged : `bool` 

1337 True when the optimizer reached convergence before 

1338 reaching the maximum number of iterations. 

1339 """ 

1340 parent.set(self.nPeaksKey, nPeaks) 

1341 parent.set(self.nChildKey, nChild) 

1342 parent.set(self.nComponentsKey, nComponents) 

1343 parent.set(self.runtimeKey, runtime) 

1344 parent.set(self.iterKey, iterations) 

1345 parent.set(self.scarletLogLKey, logL) 

1346 parent.set(self.scarletSpectrumInitKey, spectrumInit) 

1347 parent.set(self.blendConvergenceFailedFlagKey, converged) 

1348 

1349 def _addChild(self, parent, peak, catalog, scarletSource): 

1350 """Add a child to a catalog. 

1351 

1352 This creates a new child in the source catalog, 

1353 assigning it a parent id, and adding all columns 

1354 that are independent across all filter bands. 

1355 

1356 Parameters 

1357 ---------- 

1358 parent : `lsst.afw.table.source.source.SourceRecord` 

1359 The parent of the new child record. 

1360 peak : `lsst.afw.table.PeakRecord` 

1361 The peak record for the peak from the parent peak catalog. 

1362 catalog : `lsst.afw.table.source.source.SourceCatalog` 

1363 The merged `SourceCatalog` that contains parent footprints 

1364 to (potentially) deblend. 

1365 scarletSource : `scarlet.Component` 

1366 The scarlet model for the new source record. 

1367 """ 

1368 src = catalog.addNew() 

1369 for key in self.toCopyFromParent: 

1370 src.set(key, parent.get(key)) 

1371 # The peak catalog is the same for all bands, 

1372 # so we just use the first peak catalog 

1373 src.assign(peak, self.peakSchemaMapper) 

1374 src.setParent(parent.getId()) 

1375 src.set(self.nPeaksKey, 1) 

1376 # Set the psf key based on whether or not the source was 

1377 # deblended using the PointSource model. 

1378 # This key is not that useful anymore since we now keep track of 

1379 # `modelType`, but we continue to propagate it in case code downstream 

1380 # is expecting it. 

1381 src.set(self.psfKey, scarletSource.__class__.__name__ == "PointSource") 

1382 src.set(self.modelTypeKey, scarletSource.__class__.__name__) 

1383 # We set the runtime to zero so that summing up the 

1384 # runtime column will give the total time spent 

1385 # running the deblender for the catalog. 

1386 src.set(self.runtimeKey, 0) 

1387 

1388 # Set the position of the peak from the parent footprint 

1389 # This will make it easier to match the same source across 

1390 # deblenders and across observations, where the peak 

1391 # position is unlikely to change unless enough time passes 

1392 # for a source to move on the sky. 

1393 src.set(self.peakCenter, Point2I(peak["i_x"], peak["i_y"])) 

1394 src.set(self.peakIdKey, peak["id"]) 

1395 

1396 # Store the number of components for the source 

1397 src.set(self.nComponentsKey, len(scarletSource.components)) 

1398 

1399 # Propagate columns from the parent to the child 

1400 for parentColumn, childColumn in self.config.columnInheritance.items(): 

1401 src.set(childColumn, parent.get(parentColumn)) 

1402 

1403 return src