Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_deblender. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy as np 

24import time 

25 

26import scarlet 

27 

28import lsst.log 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.afw.math as afwMath 

32import lsst.afw.geom as afwGeom 

33import lsst.geom as geom 

34import lsst.afw.geom.ellipses as afwEll 

35import lsst.afw.image as afwImage 

36import lsst.afw.detection as afwDet 

37import lsst.afw.table as afwTable 

38 

39logger = lsst.log.Log.getLogger("meas.deblender.deblend") 

40 

41__all__ = 'SourceDeblendConfig', 'SourceDeblendTask', 'MultibandDeblendConfig', 'MultibandDeblendTask' 

42 

43 

44class SourceDeblendConfig(pexConfig.Config): 

45 

46 edgeHandling = pexConfig.ChoiceField( 

47 doc='What to do when a peak to be deblended is close to the edge of the image', 

48 dtype=str, default='ramp', 

49 allowed={ 

50 'clip': 'Clip the template at the edge AND the mirror of the edge.', 

51 'ramp': 'Ramp down flux at the image edge by the PSF', 

52 'noclip': 'Ignore the edge when building the symmetric template.', 

53 } 

54 ) 

55 

56 strayFluxToPointSources = pexConfig.ChoiceField( 

57 doc='When the deblender should attribute stray flux to point sources', 

58 dtype=str, default='necessary', 

59 allowed={ 

60 'necessary': 'When there is not an extended object in the footprint', 

61 'always': 'Always', 

62 'never': ('Never; stray flux will not be attributed to any deblended child ' 

63 'if the deblender thinks all peaks look like point sources'), 

64 } 

65 ) 

66 

67 assignStrayFlux = pexConfig.Field(dtype=bool, default=True, 

68 doc='Assign stray flux (not claimed by any child in the deblender) ' 

69 'to deblend children.') 

70 

71 strayFluxRule = pexConfig.ChoiceField( 

72 doc='How to split flux among peaks', 

73 dtype=str, default='trim', 

74 allowed={ 

75 'r-to-peak': '~ 1/(1+R^2) to the peak', 

76 'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. ' 

77 'CAUTION: this can be computationally expensive on large footprints!'), 

78 'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka ' 

79 'Manhattan distance)'), 

80 'trim': ('Shrink the parent footprint to pixels that are not assigned to children') 

81 } 

82 ) 

83 

84 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001, 

85 doc=('When splitting stray flux, clip fractions below ' 

86 'this value to zero.')) 

87 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=False, 

88 doc=('Chi-squared per DOF cut for deciding a source is ' 

89 'a PSF during deblending (un-shifted PSF model)')) 

90 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=False, 

91 doc=('Chi-squared per DOF cut for deciding a source is ' 

92 'PSF during deblending (shifted PSF model)')) 

93 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=False, 

94 doc=('Chi-squared per DOF cut for deciding a source is ' 

95 'a PSF during deblending (shifted PSF model #2)')) 

96 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0, 

97 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

98 " (<= 0: unlimited)")) 

99 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000, 

100 doc=("Maximum area for footprints before they are ignored as large; " 

101 "non-positive means no threshold applied")) 

102 maxFootprintSize = pexConfig.Field(dtype=int, default=0, 

103 doc=("Maximum linear dimension for footprints before they are ignored " 

104 "as large; non-positive means no threshold applied")) 

105 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0, 

106 doc=("Minimum axis ratio for footprints before they are ignored " 

107 "as large; non-positive means no threshold applied")) 

108 notDeblendedMask = pexConfig.Field(dtype=str, default="NOT_DEBLENDED", optional=True, 

109 doc="Mask name for footprints not deblended, or None") 

110 

111 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=True, 

112 doc=('Footprints smaller in width or height than this value ' 

113 'will be ignored; minimum of 2 due to PSF gradient ' 

114 'calculation.')) 

115 

116 propagateAllPeaks = pexConfig.Field(dtype=bool, default=False, 

117 doc=('Guarantee that all peaks produce a child source.')) 

118 catchFailures = pexConfig.Field( 

119 dtype=bool, default=False, 

120 doc=("If True, catch exceptions thrown by the deblender, log them, " 

121 "and set a flag on the parent, instead of letting them propagate up")) 

122 maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

123 doc="Mask planes to ignore when performing statistics") 

124 maskLimits = pexConfig.DictField( 

125 keytype=str, 

126 itemtype=float, 

127 default={}, 

128 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

129 "Sources violating this limit will not be deblended."), 

130 ) 

131 weightTemplates = pexConfig.Field( 

132 dtype=bool, default=False, 

133 doc=("If true, a least-squares fit of the templates will be done to the " 

134 "full image. The templates will be re-weighted based on this fit.")) 

135 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=False, 

136 doc=("Try to remove similar templates?")) 

137 maxTempDotProd = pexConfig.Field( 

138 dtype=float, default=0.5, 

139 doc=("If the dot product between two templates is larger than this value, we consider them to be " 

140 "describing the same object (i.e. they are degenerate). If one of the objects has been " 

141 "labeled as a PSF it will be removed, otherwise the template with the lowest value will " 

142 "be removed.")) 

143 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=True, 

144 doc="Apply a smoothing filter to all of the template images") 

145 

146## \addtogroup LSST_task_documentation 

147## \{ 

148## \page SourceDeblendTask 

149## \ref SourceDeblendTask_ "SourceDeblendTask" 

150## \copybrief SourceDeblendTask 

151## \} 

152 

153 

154class SourceDeblendTask(pipeBase.Task): 

155 """! 

156 \anchor SourceDeblendTask_ 

157 

158 \brief Split blended sources into individual sources. 

159 

160 This task has no return value; it only modifies the SourceCatalog in-place. 

161 """ 

162 ConfigClass = SourceDeblendConfig 

163 _DefaultName = "sourceDeblend" 

164 

165 def __init__(self, schema, peakSchema=None, **kwargs): 

166 """! 

167 Create the task, adding necessary fields to the given schema. 

168 

169 @param[in,out] schema Schema object for measurement fields; will be modified in-place. 

170 @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender. 

171 Any fields beyond the PeakTable minimal schema will be transferred 

172 to the main source Schema. If None, no fields will be transferred 

173 from the Peaks. 

174 @param[in] **kwargs Passed to Task.__init__. 

175 """ 

176 pipeBase.Task.__init__(self, **kwargs) 

177 self.schema = schema 

178 self.toCopyFromParent = [item.key for item in self.schema 

179 if item.field.getName().startswith("merge_footprint")] 

180 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

181 if peakSchema is None: 

182 # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one 

183 # to simplify downstream code 

184 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

185 else: 

186 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

187 for item in peakSchema: 

188 if item.key not in peakMinimalSchema: 

189 self.peakSchemaMapper.addMapping(item.key, item.field) 

190 # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't 

191 # updating this Schema in place. That's probably a design flaw, but in the meantime, 

192 # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually, 

193 # by adding the same fields to both. 

194 schema.addField(item.field) 

195 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

196 self.addSchemaKeys(schema) 

197 

198 def addSchemaKeys(self, schema): 

199 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

200 doc='Number of children this object has (defaults to 0)') 

201 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

202 doc='Deblender thought this source looked like a PSF') 

203 self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter', 

204 'If deblended-as-psf, the PSF centroid', "pixel") 

205 self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D', 

206 doc='If deblended-as-psf, the instrumental PSF flux', units='count') 

207 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

208 doc='Source had too many peaks; ' 

209 'only the brightest were included') 

210 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

211 doc='Parent footprint covered too many pixels') 

212 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

213 doc='Parent footprint was predominantly masked') 

214 

215 if self.config.catchFailures: 

216 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

217 doc="Deblending failed on source") 

218 

219 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

220 doc="Deblender skipped this source") 

221 

222 self.deblendRampedTemplateKey = schema.addField( 

223 'deblend_rampedTemplate', type='Flag', 

224 doc=('This source was near an image edge and the deblender used ' 

225 '"ramp" edge-handling.')) 

226 

227 self.deblendPatchedTemplateKey = schema.addField( 

228 'deblend_patchedTemplate', type='Flag', 

229 doc=('This source was near an image edge and the deblender used ' 

230 '"patched" edge-handling.')) 

231 

232 self.hasStrayFluxKey = schema.addField( 

233 'deblend_hasStrayFlux', type='Flag', 

234 doc=('This source was assigned some stray flux')) 

235 

236 self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in ( 

237 self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey, 

238 self.tooManyPeaksKey, self.tooBigKey))) 

239 

240 @pipeBase.timeMethod 

241 def run(self, exposure, sources): 

242 """! 

243 Get the psf from the provided exposure and then run deblend(). 

244 

245 @param[in] exposure Exposure to process 

246 @param[in,out] sources SourceCatalog containing sources detected on this exposure. 

247 

248 @return None 

249 """ 

250 psf = exposure.getPsf() 

251 assert sources.getSchema() == self.schema 

252 self.deblend(exposure, sources, psf) 

253 

254 def _getPsfFwhm(self, psf, bbox): 

255 # It should be easier to get a PSF's fwhm; 

256 # https://dev.lsstcorp.org/trac/ticket/3030 

257 return psf.computeShape().getDeterminantRadius() * 2.35 

258 

259 @pipeBase.timeMethod 

260 def deblend(self, exposure, srcs, psf): 

261 """! 

262 Deblend. 

263 

264 @param[in] exposure Exposure to process 

265 @param[in,out] srcs SourceCatalog containing sources detected on this exposure. 

266 @param[in] psf PSF 

267 

268 @return None 

269 """ 

270 self.log.info("Deblending %d sources" % len(srcs)) 

271 

272 from lsst.meas.deblender.baseline import deblend 

273 

274 # find the median stdev in the image... 

275 mi = exposure.getMaskedImage() 

276 statsCtrl = afwMath.StatisticsControl() 

277 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes)) 

278 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl) 

279 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN)) 

280 self.log.trace('sigma1: %g', sigma1) 

281 

282 n0 = len(srcs) 

283 nparents = 0 

284 for i, src in enumerate(srcs): 

285 # t0 = time.clock() 

286 

287 fp = src.getFootprint() 

288 pks = fp.getPeaks() 

289 

290 # Since we use the first peak for the parent object, we should propagate its flags 

291 # to the parent source. 

292 src.assign(pks[0], self.peakSchemaMapper) 

293 

294 if len(pks) < 2: 

295 continue 

296 

297 if self.isLargeFootprint(fp): 

298 src.set(self.tooBigKey, True) 

299 self.skipParent(src, mi.getMask()) 

300 self.log.warn('Parent %i: skipping large footprint (area: %i)', 

301 int(src.getId()), int(fp.getArea())) 

302 continue 

303 if self.isMasked(fp, exposure.getMaskedImage().getMask()): 

304 src.set(self.maskedKey, True) 

305 self.skipParent(src, mi.getMask()) 

306 self.log.warn('Parent %i: skipping masked footprint (area: %i)', 

307 int(src.getId()), int(fp.getArea())) 

308 continue 

309 

310 nparents += 1 

311 bb = fp.getBBox() 

312 psf_fwhm = self._getPsfFwhm(psf, bb) 

313 

314 self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks)) 

315 

316 self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1) 

317 npre = len(srcs) 

318 

319 # This should really be set in deblend, but deblend doesn't have access to the src 

320 src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks) 

321 

322 try: 

323 res = deblend( 

324 fp, mi, psf, psf_fwhm, sigma1=sigma1, 

325 psfChisqCut1=self.config.psfChisq1, 

326 psfChisqCut2=self.config.psfChisq2, 

327 psfChisqCut2b=self.config.psfChisq2b, 

328 maxNumberOfPeaks=self.config.maxNumberOfPeaks, 

329 strayFluxToPointSources=self.config.strayFluxToPointSources, 

330 assignStrayFlux=self.config.assignStrayFlux, 

331 strayFluxAssignment=self.config.strayFluxRule, 

332 rampFluxAtEdge=(self.config.edgeHandling == 'ramp'), 

333 patchEdges=(self.config.edgeHandling == 'noclip'), 

334 tinyFootprintSize=self.config.tinyFootprintSize, 

335 clipStrayFluxFraction=self.config.clipStrayFluxFraction, 

336 weightTemplates=self.config.weightTemplates, 

337 removeDegenerateTemplates=self.config.removeDegenerateTemplates, 

338 maxTempDotProd=self.config.maxTempDotProd, 

339 medianSmoothTemplate=self.config.medianSmoothTemplate 

340 ) 

341 if self.config.catchFailures: 

342 src.set(self.deblendFailedKey, False) 

343 except Exception as e: 

344 if self.config.catchFailures: 

345 self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e)) 

346 src.set(self.deblendFailedKey, True) 

347 import traceback 

348 traceback.print_exc() 

349 continue 

350 else: 

351 raise 

352 

353 kids = [] 

354 nchild = 0 

355 for j, peak in enumerate(res.deblendedParents[0].peaks): 

356 heavy = peak.getFluxPortion() 

357 if heavy is None or peak.skip: 

358 src.set(self.deblendSkippedKey, True) 

359 if not self.config.propagateAllPeaks: 

360 # Don't care 

361 continue 

362 # We need to preserve the peak: make sure we have enough info to create a minimal 

363 # child src 

364 self.log.trace("Peak at (%i,%i) failed. Using minimal default info for child.", 

365 pks[j].getIx(), pks[j].getIy()) 

366 if heavy is None: 

367 # copy the full footprint and strip out extra peaks 

368 foot = afwDet.Footprint(src.getFootprint()) 

369 peakList = foot.getPeaks() 

370 peakList.clear() 

371 peakList.append(peak.peak) 

372 zeroMimg = afwImage.MaskedImageF(foot.getBBox()) 

373 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg) 

374 if peak.deblendedAsPsf: 

375 if peak.psfFitFlux is None: 

376 peak.psfFitFlux = 0.0 

377 if peak.psfFitCenter is None: 

378 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy()) 

379 

380 assert(len(heavy.getPeaks()) == 1) 

381 

382 src.set(self.deblendSkippedKey, False) 

383 child = srcs.addNew() 

384 nchild += 1 

385 for key in self.toCopyFromParent: 

386 child.set(key, src.get(key)) 

387 child.assign(heavy.getPeaks()[0], self.peakSchemaMapper) 

388 child.setParent(src.getId()) 

389 child.setFootprint(heavy) 

390 child.set(self.psfKey, peak.deblendedAsPsf) 

391 child.set(self.hasStrayFluxKey, peak.strayFlux is not None) 

392 if peak.deblendedAsPsf: 

393 (cx, cy) = peak.psfFitCenter 

394 child.set(self.psfCenterKey, geom.Point2D(cx, cy)) 

395 child.set(self.psfFluxKey, peak.psfFitFlux) 

396 child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate) 

397 child.set(self.deblendPatchedTemplateKey, peak.patched) 

398 kids.append(child) 

399 

400 # Child footprints may extend beyond the full extent of their parent's which 

401 # results in a failure of the replace-by-noise code to reinstate these pixels 

402 # to their original values. The following updates the parent footprint 

403 # in-place to ensure it contains the full union of itself and all of its 

404 # children's footprints. 

405 spans = src.getFootprint().spans 

406 for child in kids: 

407 spans = spans.union(child.getFootprint().spans) 

408 src.getFootprint().setSpans(spans) 

409 

410 src.set(self.nChildKey, nchild) 

411 

412 self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res) 

413 # print('Deblending parent id', src.getId(), 'took', time.clock() - t0) 

414 

415 n1 = len(srcs) 

416 self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' 

417 % (n0, nparents, n1-n0, n1)) 

418 

419 def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1): 

420 pass 

421 

422 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res): 

423 pass 

424 

425 def isLargeFootprint(self, footprint): 

426 """Returns whether a Footprint is large 

427 

428 'Large' is defined by thresholds on the area, size and axis ratio. 

429 These may be disabled independently by configuring them to be non-positive. 

430 

431 This is principally intended to get rid of satellite streaks, which the 

432 deblender or other downstream processing can have trouble dealing with 

433 (e.g., multiple large HeavyFootprints can chew up memory). 

434 """ 

435 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

436 return True 

437 if self.config.maxFootprintSize > 0: 

438 bbox = footprint.getBBox() 

439 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

440 return True 

441 if self.config.minFootprintAxisRatio > 0: 

442 axes = afwEll.Axes(footprint.getShape()) 

443 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

444 return True 

445 return False 

446 

447 def isMasked(self, footprint, mask): 

448 """Returns whether the footprint violates the mask limits""" 

449 size = float(footprint.getArea()) 

450 for maskName, limit in self.config.maskLimits.items(): 

451 maskVal = mask.getPlaneBitMask(maskName) 

452 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels 

453 if (size - unmaskedSpan.getArea())/size > limit: 

454 return True 

455 return False 

456 

457 def skipParent(self, source, mask): 

458 """Indicate that the parent source is not being deblended 

459 

460 We set the appropriate flags and mask. 

461 

462 @param source The source to flag as skipped 

463 @param mask The mask to update 

464 """ 

465 fp = source.getFootprint() 

466 source.set(self.deblendSkippedKey, True) 

467 source.set(self.nChildKey, len(fp.getPeaks())) # It would have this many if we deblended them all 

468 if self.config.notDeblendedMask: 

469 mask.addMaskPlane(self.config.notDeblendedMask) 

470 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask)) 

471 

472 

473class MultibandDeblendConfig(pexConfig.Config): 

474 """MultibandDeblendConfig 

475 

476 Configuration for the multiband deblender. 

477 The parameters are organized by the parameter types, which are 

478 - Stopping Criteria: Used to determine if the fit has converged 

479 - Position Fitting Criteria: Used to fit the positions of the peaks 

480 - Constraints: Used to apply constraints to the peaks and their components 

481 - Other: Parameters that don't fit into the above categories 

482 """ 

483 # Stopping Criteria 

484 maxIter = pexConfig.Field(dtype=int, default=200, 

485 doc=("Maximum number of iterations to deblend a single parent")) 

486 relativeError = pexConfig.Field(dtype=float, default=1e-3, 

487 doc=("Relative error to use when determining stopping criteria")) 

488 

489 # Blend Configuration options 

490 minTranslation = pexConfig.Field(dtype=float, default=1e-3, 

491 doc=("A peak must be updated by at least 'minTranslation' (pixels)" 

492 "or no update is performed." 

493 "This field is ignored if fitPositions is False.")) 

494 refinementSkip = pexConfig.Field(dtype=int, default=10, 

495 doc=("If fitPositions is True, the positions and box sizes are" 

496 "updated on every 'refinementSkip' iterations.")) 

497 translationMethod = pexConfig.Field(dtype=str, default="default", 

498 doc=("Method to use for fitting translations." 

499 "Currently 'default' is the only available option," 

500 "which performs a linear fit, but it is possible that we" 

501 "will use galsim or some other method as a future option")) 

502 edgeFluxThresh = pexConfig.Field(dtype=float, default=1.0, 

503 doc=("Boxes are resized when the flux at an edge is " 

504 "> edgeFluxThresh * background RMS")) 

505 exactLipschitz = pexConfig.Field(dtype=bool, default=False, 

506 doc=("Calculate exact Lipschitz constant in every step" 

507 "(True) or only calculate the approximate" 

508 "Lipschitz constant with significant changes in A,S" 

509 "(False)")) 

510 stepSlack = pexConfig.Field(dtype=float, default=0.2, 

511 doc=("A fractional measure of how much a value (like the exactLipschitz)" 

512 "can change before it needs to be recalculated." 

513 "This must be between 0 and 1.")) 

514 

515 # Constraints 

516 constraints = pexConfig.Field(dtype=str, default="1,+,S,M", 

517 doc=("List of constraints to use for each object" 

518 "(order does not matter)" 

519 "Current options are all used by default:\n" 

520 "S: symmetry\n" 

521 "M: monotonicity\n" 

522 "1: normalized SED to unity" 

523 "+: non-negative morphology")) 

524 symmetryThresh = pexConfig.Field(dtype=float, default=1.0, 

525 doc=("Strictness of symmetry, from" 

526 "0 (no symmetry enforced) to" 

527 "1 (perfect symmetry required)." 

528 "If 'S' is not in `constraints`, this argument is ignored")) 

529 l0Thresh = pexConfig.Field(dtype=float, default=np.nan, 

530 doc=("L0 threshold. NaN results in no L0 penalty.")) 

531 l1Thresh = pexConfig.Field(dtype=float, default=np.nan, 

532 doc=("L1 threshold. NaN results in no L1 penalty.")) 

533 tvxThresh = pexConfig.Field(dtype=float, default=np.nan, 

534 doc=("Threshold for TV (total variation) constraint in the x-direction." 

535 "NaN results in no TVx penalty.")) 

536 tvyThresh = pexConfig.Field(dtype=float, default=np.nan, 

537 doc=("Threshold for TV (total variation) constraint in the y-direction." 

538 "NaN results in no TVy penalty.")) 

539 

540 # Other scarlet paremeters 

541 useWeights = pexConfig.Field(dtype=bool, default=False, doc="Use inverse variance as deblender weights") 

542 bgScale = pexConfig.Field( 

543 dtype=float, default=0.5, 

544 doc=("Fraction of background RMS level to use as a" 

545 "cutoff for defining the background of the image" 

546 "This is used to initialize the model for each source" 

547 "and to set the size of the bounding box for each source" 

548 "every `refinementSkip` iteration.")) 

549 usePsfConvolution = pexConfig.Field( 

550 dtype=bool, default=True, 

551 doc=("Whether or not to convolve the morphology with the" 

552 "PSF in each band or use the same morphology in all bands")) 

553 saveTemplates = pexConfig.Field( 

554 dtype=bool, default=True, 

555 doc="Whether or not to save the SEDs and templates") 

556 processSingles = pexConfig.Field( 

557 dtype=bool, default=False, 

558 doc="Whether or not to process isolated sources in the deblender") 

559 badMask = pexConfig.Field( 

560 dtype=str, default="BAD,CR,NO_DATA,SAT,SUSPECT", 

561 doc="Whether or not to process isolated sources in the deblender") 

562 # Old deblender parameters used in this implementation (some of which might be removed later) 

563 

564 maxNumberOfPeaks = pexConfig.Field( 

565 dtype=int, default=0, 

566 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

567 " (<= 0: unlimited)")) 

568 maxFootprintArea = pexConfig.Field( 

569 dtype=int, default=1000000, 

570 doc=("Maximum area for footprints before they are ignored as large; " 

571 "non-positive means no threshold applied")) 

572 maxFootprintSize = pexConfig.Field( 

573 dtype=int, default=0, 

574 doc=("Maximum linear dimension for footprints before they are ignored " 

575 "as large; non-positive means no threshold applied")) 

576 minFootprintAxisRatio = pexConfig.Field( 

577 dtype=float, default=0.0, 

578 doc=("Minimum axis ratio for footprints before they are ignored " 

579 "as large; non-positive means no threshold applied")) 

580 notDeblendedMask = pexConfig.Field( 

581 dtype=str, default="NOT_DEBLENDED", optional=True, 

582 doc="Mask name for footprints not deblended, or None") 

583 

584 tinyFootprintSize = pexConfig.RangeField( 

585 dtype=int, default=2, min=2, inclusiveMin=True, 

586 doc=('Footprints smaller in width or height than this value will ' 

587 'be ignored; minimum of 2 due to PSF gradient calculation.')) 

588 catchFailures = pexConfig.Field( 

589 dtype=bool, default=False, 

590 doc=("If True, catch exceptions thrown by the deblender, log them, " 

591 "and set a flag on the parent, instead of letting them propagate up")) 

592 propagateAllPeaks = pexConfig.Field(dtype=bool, default=False, 

593 doc=('Guarantee that all peaks produce a child source.')) 

594 maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

595 doc="Mask planes to ignore when performing statistics") 

596 maskLimits = pexConfig.DictField( 

597 keytype=str, 

598 itemtype=float, 

599 default={}, 

600 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

601 "Sources violating this limit will not be deblended."), 

602 ) 

603 

604 edgeHandling = pexConfig.ChoiceField( 

605 doc='What to do when a peak to be deblended is close to the edge of the image', 

606 dtype=str, default='noclip', 

607 allowed={ 

608 'clip': 'Clip the template at the edge AND the mirror of the edge.', 

609 'ramp': 'Ramp down flux at the image edge by the PSF', 

610 'noclip': 'Ignore the edge when building the symmetric template.', 

611 } 

612 ) 

613 

614 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=False, 

615 doc="Apply a smoothing filter to all of the template images") 

616 medianFilterHalfsize = pexConfig.Field(dtype=float, default=2, 

617 doc=('Half size of the median smoothing filter')) 

618 clipFootprintToNonzero = pexConfig.Field(dtype=bool, default=False, 

619 doc=("Clip non-zero spans in the footprints")) 

620 

621 conserveFlux = pexConfig.Field(dtype=bool, default=True, 

622 doc=("Reapportion flux to the footprints so that flux is conserved")) 

623 weightTemplates = pexConfig.Field( 

624 dtype=bool, default=False, 

625 doc=("If true, a least-squares fit of the templates will be done to the " 

626 "full image. The templates will be re-weighted based on this fit.")) 

627 strayFluxToPointSources = pexConfig.ChoiceField( 

628 doc='When the deblender should attribute stray flux to point sources', 

629 dtype=str, default='necessary', 

630 allowed={ 

631 'necessary': 'When there is not an extended object in the footprint', 

632 'always': 'Always', 

633 'never': ('Never; stray flux will not be attributed to any deblended child ' 

634 'if the deblender thinks all peaks look like point sources'), 

635 } 

636 ) 

637 

638 assignStrayFlux = pexConfig.Field(dtype=bool, default=True, 

639 doc='Assign stray flux (not claimed by any child in the deblender) ' 

640 'to deblend children.') 

641 

642 strayFluxRule = pexConfig.ChoiceField( 

643 doc='How to split flux among peaks', 

644 dtype=str, default='trim', 

645 allowed={ 

646 'r-to-peak': '~ 1/(1+R^2) to the peak', 

647 'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. ' 

648 'CAUTION: this can be computationally expensive on large footprints!'), 

649 'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka ' 

650 'Manhattan distance)'), 

651 'trim': ('Shrink the parent footprint to pixels that are not assigned to children') 

652 } 

653 ) 

654 

655 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001, 

656 doc=('When splitting stray flux, clip fractions below ' 

657 'this value to zero.')) 

658 getTemplateSum = pexConfig.Field(dtype=bool, default=False, 

659 doc=("As part of the flux calculation, the sum of the templates is" 

660 "calculated. If 'getTemplateSum==True' then the sum of the" 

661 "templates is stored in the result (a 'PerFootprint').")) 

662 

663 

664class MultibandDeblendTask(pipeBase.Task): 

665 """MultibandDeblendTask 

666 

667 Split blended sources into individual sources. 

668 

669 This task has no return value; it only modifies the SourceCatalog in-place. 

670 """ 

671 ConfigClass = MultibandDeblendConfig 

672 _DefaultName = "multibandDeblend" 

673 

674 def __init__(self, schema, peakSchema=None, **kwargs): 

675 """Create the task, adding necessary fields to the given schema. 

676 

677 Parameters 

678 ---------- 

679 schema: `lsst.afw.table.schema.schema.Schema` 

680 Schema object for measurement fields; will be modified in-place. 

681 peakSchema: `lsst.afw.table.schema.schema.Schema` 

682 Schema of Footprint Peaks that will be passed to the deblender. 

683 Any fields beyond the PeakTable minimal schema will be transferred 

684 to the main source Schema. If None, no fields will be transferred 

685 from the Peaks. 

686 filters: list of str 

687 Names of the filters used for the eposures. This is needed to store the SED as a field 

688 **kwargs 

689 Passed to Task.__init__. 

690 """ 

691 from lsst.meas.deblender import plugins 

692 

693 pipeBase.Task.__init__(self, **kwargs) 

694 if not self.config.conserveFlux and not self.config.saveTemplates: 

695 raise ValueError("Either `conserveFlux` or `saveTemplates` must be True") 

696 

697 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

698 if peakSchema is None: 

699 # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one 

700 # to simplify downstream code 

701 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

702 else: 

703 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

704 for item in peakSchema: 

705 if item.key not in peakMinimalSchema: 

706 self.peakSchemaMapper.addMapping(item.key, item.field) 

707 # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't 

708 # updating this Schema in place. That's probably a design flaw, but in the meantime, 

709 # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually, 

710 # by adding the same fields to both. 

711 schema.addField(item.field) 

712 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

713 self._addSchemaKeys(schema) 

714 self.schema = schema 

715 

716 # Create the plugins for multiband deblending using the Config options 

717 

718 # Basic deblender configuration 

719 config = scarlet.config.Config( 

720 center_min_dist=self.config.minTranslation, 

721 edge_flux_thresh=self.config.edgeFluxThresh, 

722 exact_lipschitz=self.config.exactLipschitz, 

723 refine_skip=self.config.refinementSkip, 

724 slack=self.config.stepSlack, 

725 ) 

726 if self.config.translationMethod != "default": 

727 err = "Currently the only supported translationMethod is 'default', you entered '{0}'" 

728 raise NotImplementedError(err.format(self.config.translationMethod)) 

729 

730 # If the default constraints are not used, set the constraints for 

731 # all of the sources 

732 constraints = None 

733 _constraints = self.config.constraints.split(",") 

734 if (sorted(_constraints) != ['+', '1', 'M', 'S'] or 

735 ~np.isnan(self.config.l0Thresh) or 

736 ~np.isnan(self.config.l1Thresh)): 

737 constraintDict = { 

738 "+": scarlet.constraint.PositivityConstraint, 

739 "1": scarlet.constraint.SimpleConstraint, 

740 "M": scarlet.constraint.DirectMonotonicityConstraint(use_nearest=False), 

741 "S": scarlet.constraint.DirectSymmetryConstraint(sigma=self.config.symmetryThresh) 

742 } 

743 for c in _constraints: 

744 if constraints is None: 

745 constraints = [constraintDict[c]] 

746 else: 

747 constraints += [constraintDict[c]] 

748 if constraints is None: 

749 constraints = scarlet.constraint.MinimalConstraint() 

750 if ~np.isnan(self.config.l0Thresh): 

751 constraints += [scarlet.constraint.L0Constraint(self.config.l0Thresh)] 

752 if ~np.isnan(self.config.l1Thresh): 

753 constraints += [scarlet.constraint.L1Constraint(self.config.l1Thresh)] 

754 if ~np.isnan(self.config.tvxThresh): 

755 constraints += [scarlet.constraint.TVxConstraint(self.config.tvxThresh)] 

756 if ~np.isnan(self.config.tvyThresh): 

757 constraints += [scarlet.constraint.TVyConstraint(self.config.tvyThresh)] 

758 

759 multiband_plugin = plugins.DeblenderPlugin( 

760 plugins.buildMultibandTemplates, 

761 useWeights=self.config.useWeights, 

762 usePsf=self.config.usePsfConvolution, 

763 constraints=constraints, 

764 config=config, 

765 maxIter=self.config.maxIter, 

766 bgScale=self.config.bgScale, 

767 relativeError=self.config.relativeError, 

768 badMask=self.config.badMask.split(","), 

769 ) 

770 self.plugins = [multiband_plugin] 

771 

772 # Plugins from the old deblender for post-template processing 

773 # (see lsst.meas_deblender.baseline.deblend) 

774 if self.config.edgeHandling == 'ramp': 

775 self.plugins.append(plugins.DeblenderPlugin(plugins.rampFluxAtEdge, patchEdges=False)) 

776 if self.config.medianSmoothTemplate: 

777 self.plugins.append(plugins.DeblenderPlugin( 

778 plugins.medianSmoothTemplates, 

779 medianFilterHalfsize=self.config.medianFilterHalfsize)) 

780 if self.config.clipFootprintToNonzero: 

781 self.plugins.append(plugins.DeblenderPlugin(plugins.clipFootprintsToNonzero)) 

782 if self.config.conserveFlux: 

783 if self.config.weightTemplates: 

784 self.plugins.append(plugins.DeblenderPlugin(plugins.weightTemplates)) 

785 self.plugins.append(plugins.DeblenderPlugin( 

786 plugins.apportionFlux, 

787 clipStrayFluxFraction=self.config.clipStrayFluxFraction, 

788 assignStrayFlux=self.config.assignStrayFlux, 

789 strayFluxAssignment=self.config.strayFluxRule, 

790 strayFluxToPointSources=self.config.strayFluxToPointSources, 

791 getTemplateSum=self.config.getTemplateSum)) 

792 

793 def _addSchemaKeys(self, schema): 

794 """Add deblender specific keys to the schema 

795 """ 

796 self.runtimeKey = schema.addField('runtime', type=np.float32, doc='runtime in ms') 

797 # Keys from old Deblender that might be kept in the new deblender 

798 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

799 doc='Number of children this object has (defaults to 0)') 

800 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

801 doc='Deblender thought this source looked like a PSF') 

802 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

803 doc='Source had too many peaks; ' 

804 'only the brightest were included') 

805 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

806 doc='Parent footprint covered too many pixels') 

807 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

808 doc='Parent footprint was predominantly masked') 

809 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

810 doc="Deblending failed on source") 

811 

812 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

813 doc="Deblender skipped this source") 

814 

815 # Keys from the old Deblender that some measruement tasks require. 

816 # TODO: Remove these whem the old deblender is removed 

817 self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter', 

818 'If deblended-as-psf, the PSF centroid', "pixel") 

819 self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D', 

820 doc='If deblended-as-psf, the instrumental PSF flux', units='count') 

821 self.deblendRampedTemplateKey = schema.addField( 

822 'deblend_rampedTemplate', type='Flag', 

823 doc=('This source was near an image edge and the deblender used ' 

824 '"ramp" edge-handling.')) 

825 

826 self.deblendPatchedTemplateKey = schema.addField( 

827 'deblend_patchedTemplate', type='Flag', 

828 doc=('This source was near an image edge and the deblender used ' 

829 '"patched" edge-handling.')) 

830 

831 self.hasStrayFluxKey = schema.addField( 

832 'deblend_hasStrayFlux', type='Flag', 

833 doc=('This source was assigned some stray flux')) 

834 

835 self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in ( 

836 self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey, 

837 self.tooManyPeaksKey, self.tooBigKey))) 

838 

839 @pipeBase.timeMethod 

840 def run(self, mExposure, mergedSources): 

841 """Get the psf from each exposure and then run deblend(). 

842 

843 Parameters 

844 ---------- 

845 mExposure: `MultibandExposure` 

846 The exposures should be co-added images of the same 

847 shape and region of the sky. 

848 mergedSources: `SourceCatalog` 

849 The merged `SourceCatalog` that contains parent footprints 

850 to (potentially) deblend. 

851 

852 Returns 

853 ------- 

854 fluxCatalogs: dict or None 

855 Keys are the names of the filters and the values are 

856 `lsst.afw.table.source.source.SourceCatalog`'s. 

857 These are the flux-conserved catalogs with heavy footprints with 

858 the image data weighted by the multiband templates. 

859 If `self.config.conserveFlux` is `False`, then this item will be None 

860 templateCatalogs: dict or None 

861 Keys are the names of the filters and the values are 

862 `lsst.afw.table.source.source.SourceCatalog`'s. 

863 These are catalogs with heavy footprints that are the templates 

864 created by the multiband templates. 

865 If `self.config.saveTemplates` is `False`, then this item will be None 

866 """ 

867 psfs = {f: mExposure[f].getPsf() for f in mExposure.filters} 

868 return self.deblend(mExposure, mergedSources, psfs) 

869 

870 def _getPsfFwhm(self, psf, bbox): 

871 return psf.computeShape().getDeterminantRadius() * 2.35 

872 

873 def _addChild(self, parentId, peak, sources, heavy): 

874 """Add a child to a catalog 

875 

876 This creates a new child in the source catalog, 

877 assigning it a parent id, adding a footprint, 

878 and setting all appropriate flags based on the 

879 deblender result. 

880 """ 

881 assert len(heavy.getPeaks()) == 1 

882 src = sources.addNew() 

883 src.assign(heavy.getPeaks()[0], self.peakSchemaMapper) 

884 src.setParent(parentId) 

885 src.setFootprint(heavy) 

886 src.set(self.psfKey, peak.deblendedAsPsf) 

887 src.set(self.hasStrayFluxKey, peak.strayFlux is not None) 

888 src.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate) 

889 src.set(self.deblendPatchedTemplateKey, peak.patched) 

890 src.set(self.runtimeKey, 0) 

891 return src 

892 

893 @pipeBase.timeMethod 

894 def deblend(self, mExposure, sources, psfs): 

895 """Deblend a data cube of multiband images 

896 

897 Parameters 

898 ---------- 

899 mExposure: `MultibandExposure` 

900 The exposures should be co-added images of the same 

901 shape and region of the sky. 

902 sources: `SourceCatalog` 

903 The merged `SourceCatalog` that contains parent footprints 

904 to (potentially) deblend. 

905 psfs: dict 

906 Keys are the names of the filters 

907 (should be the same as `mExposure.filters`) 

908 and the values are the PSFs in each band. 

909 

910 Returns 

911 ------- 

912 fluxCatalogs: dict or None 

913 Keys are the names of the filters and the values are 

914 `lsst.afw.table.source.source.SourceCatalog`'s. 

915 These are the flux-conserved catalogs with heavy footprints with 

916 the image data weighted by the multiband templates. 

917 If `self.config.conserveFlux` is `False`, then this item will be None 

918 templateCatalogs: dict or None 

919 Keys are the names of the filters and the values are 

920 `lsst.afw.table.source.source.SourceCatalog`'s. 

921 These are catalogs with heavy footprints that are the templates 

922 created by the multiband templates. 

923 If `self.config.saveTemplates` is `False`, then this item will be None 

924 """ 

925 from lsst.meas.deblender.baseline import newDeblend 

926 

927 if tuple(psfs.keys()) != mExposure.filters: 

928 msg = "PSF keys must be the same as mExposure.filters ({0}), got {1}" 

929 raise ValueError(msg.format(mExposure.filters, psfs.keys())) 

930 

931 filters = mExposure.filters 

932 mMaskedImage = afwImage.MultibandMaskedImage(filters=mExposure.filters, image=mExposure.image, 

933 mask=mExposure.mask, variance=mExposure.variance) 

934 self.log.info("Deblending {0} sources in {1} exposures".format(len(sources), len(mExposure))) 

935 

936 # find the median stdev in each image 

937 sigmas = {} 

938 for f in filters: 

939 exposure = mExposure[f] 

940 mi = exposure.getMaskedImage() 

941 statsCtrl = afwMath.StatisticsControl() 

942 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes)) 

943 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl) 

944 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN)) 

945 self.log.trace('Exposure {0}, sigma1: {1}'.format(f, sigma1)) 

946 sigmas[f] = sigma1 

947 

948 # Create the output catalogs 

949 if self.config.conserveFlux: 

950 fluxCatalogs = {} 

951 for f in filters: 

952 _catalog = afwTable.SourceCatalog(sources.table.clone()) 

953 _catalog.extend(sources) 

954 fluxCatalogs[f] = _catalog 

955 else: 

956 fluxCatalogs = None 

957 if self.config.saveTemplates: 

958 templateCatalogs = {} 

959 for f in filters: 

960 _catalog = afwTable.SourceCatalog(sources.table.clone()) 

961 _catalog.extend(sources) 

962 templateCatalogs[f] = _catalog 

963 else: 

964 templateCatalogs = None 

965 

966 n0 = len(sources) 

967 nparents = 0 

968 for pk, src in enumerate(sources): 

969 foot = src.getFootprint() 

970 logger.info("id: {0}".format(src["id"])) 

971 peaks = foot.getPeaks() 

972 

973 # Since we use the first peak for the parent object, we should propagate its flags 

974 # to the parent source. 

975 src.assign(peaks[0], self.peakSchemaMapper) 

976 

977 # Block of Skipping conditions 

978 if len(peaks) < 2 and not self.config.processSingles: 

979 for f in filters: 

980 if self.config.saveTemplates: 

981 templateCatalogs[f][pk].set(self.runtimeKey, 0) 

982 if self.config.conserveFlux: 

983 fluxCatalogs[f][pk].set(self.runtimeKey, 0) 

984 continue 

985 if self.isLargeFootprint(foot): 

986 src.set(self.tooBigKey, True) 

987 self.skipParent(src, [mi.getMask() for mi in mMaskedImage]) 

988 self.log.warn('Parent %i: skipping large footprint (area: %i)', 

989 int(src.getId()), int(foot.getArea())) 

990 continue 

991 if self.isMasked(foot, exposure.getMaskedImage().getMask()): 

992 src.set(self.maskedKey, True) 

993 self.skipParent(src, mi.getMask()) 

994 self.log.trace('Parent %i: skipping masked footprint (area: %i)', 

995 int(src.getId()), int(foot.getArea())) 

996 continue 

997 if len(peaks) > self.config.maxNumberOfPeaks: 

998 src.set(self.tooManyPeaksKey, True) 

999 msg = 'Parent {0}: Too many peaks, using the first {1} peaks' 

1000 self.log.trace(msg.format(int(src.getId()), self.config.maxNumberOfPeaks)) 

1001 

1002 nparents += 1 

1003 bbox = foot.getBBox() 

1004 psf_fwhms = {f: self._getPsfFwhm(psf, bbox) for f, psf in psfs.items()} 

1005 self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(peaks)) 

1006 self.preSingleDeblendHook(mExposure.singles, sources, pk, foot, psfs, psf_fwhms, sigmas) 

1007 npre = len(sources) 

1008 # Run the deblender 

1009 try: 

1010 t0 = time.time() 

1011 # Build the parameter lists with the same ordering 

1012 images = mMaskedImage[:, bbox] 

1013 psf_list = [psfs[f] for f in filters] 

1014 fwhm_list = [psf_fwhms[f] for f in filters] 

1015 avgNoise = [sigmas[f] for f in filters] 

1016 

1017 result = newDeblend(debPlugins=self.plugins, 

1018 footprint=foot, 

1019 mMaskedImage=images, 

1020 psfs=psf_list, 

1021 psfFwhms=fwhm_list, 

1022 avgNoise=avgNoise, 

1023 maxNumberOfPeaks=self.config.maxNumberOfPeaks) 

1024 tf = time.time() 

1025 runtime = (tf-t0)*1000 

1026 if result.failed: 

1027 src.set(self.deblendFailedKey, False) 

1028 src.set(self.runtimeKey, 0) 

1029 continue 

1030 except Exception as e: 

1031 if self.config.catchFailures: 

1032 self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e)) 

1033 src.set(self.deblendFailedKey, True) 

1034 src.set(self.runtimeKey, 0) 

1035 import traceback 

1036 traceback.print_exc() 

1037 continue 

1038 else: 

1039 raise 

1040 

1041 # Add the merged source as a parent in the catalog for each band 

1042 templateParents = {} 

1043 fluxParents = {} 

1044 parentId = src.getId() 

1045 for f in filters: 

1046 if self.config.saveTemplates: 

1047 templateParents[f] = templateCatalogs[f][pk] 

1048 templateParents[f].set(self.runtimeKey, runtime) 

1049 if self.config.conserveFlux: 

1050 fluxParents[f] = fluxCatalogs[f][pk] 

1051 fluxParents[f].set(self.runtimeKey, runtime) 

1052 

1053 # Add each source to the catalogs in each band 

1054 templateSpans = {f: afwGeom.SpanSet() for f in filters} 

1055 fluxSpans = {f: afwGeom.SpanSet() for f in filters} 

1056 nchild = 0 

1057 for j, multiPeak in enumerate(result.peaks): 

1058 heavy = {f: peak.getFluxPortion() for f, peak in multiPeak.deblendedPeaks.items()} 

1059 no_flux = all([v is None for v in heavy.values()]) 

1060 skip_peak = all([peak.skip for peak in multiPeak.deblendedPeaks.values()]) 

1061 if no_flux or skip_peak: 

1062 src.set(self.deblendSkippedKey, True) 

1063 if not self.config.propagateAllPeaks: 

1064 # We don't care 

1065 continue 

1066 # We need to preserve the peak: make sure we have enough info to create a minimal 

1067 # child src 

1068 msg = "Peak at {0} failed deblending. Using minimal default info for child." 

1069 self.log.trace(msg.format(multiPeak.x, multiPeak.y)) 

1070 

1071 # copy the full footprint and strip out extra peaks 

1072 pfoot = afwDet.Footprint(foot) 

1073 peakList = pfoot.getPeaks() 

1074 peakList.clear() 

1075 pfoot.addPeak(multiPeak.x, multiPeak.y, 0) 

1076 zeroMimg = afwImage.MaskedImageF(pfoot.getBBox()) 

1077 for f in filters: 

1078 heavy[f] = afwDet.makeHeavyFootprint(pfoot, zeroMimg) 

1079 else: 

1080 src.set(self.deblendSkippedKey, False) 

1081 

1082 # Add the peak to the source catalog in each band 

1083 for f in filters: 

1084 if len(heavy[f].getPeaks()) != 1: 

1085 err = "Heavy footprint should have a single peak, got {0}" 

1086 raise ValueError(err.format(len(heavy[f].getPeaks()))) 

1087 peak = multiPeak.deblendedPeaks[f] 

1088 if self.config.saveTemplates: 

1089 cat = templateCatalogs[f] 

1090 tfoot = peak.templateFootprint 

1091 timg = afwImage.MaskedImageF(peak.templateImage) 

1092 tHeavy = afwDet.makeHeavyFootprint(tfoot, timg) 

1093 child = self._addChild(parentId, peak, cat, tHeavy) 

1094 if parentId == 0: 

1095 child.setId(src.getId()) 

1096 child.set(self.runtimeKey, runtime) 

1097 else: 

1098 templateSpans[f] = templateSpans[f].union(tHeavy.getSpans()) 

1099 if self.config.conserveFlux: 

1100 cat = fluxCatalogs[f] 

1101 child = self._addChild(parentId, peak, cat, heavy[f]) 

1102 if parentId == 0: 

1103 child.setId(src.getId()) 

1104 child.set(self.runtimeKey, runtime) 

1105 else: 

1106 fluxSpans[f] = fluxSpans[f].union(heavy[f].getSpans()) 

1107 nchild += 1 

1108 

1109 # Child footprints may extend beyond the full extent of their parent's which 

1110 # results in a failure of the replace-by-noise code to reinstate these pixels 

1111 # to their original values. The following updates the parent footprint 

1112 # in-place to ensure it contains the full union of itself and all of its 

1113 # children's footprints. 

1114 for f in filters: 

1115 if self.config.saveTemplates: 

1116 templateParents[f].set(self.nChildKey, nchild) 

1117 templateParents[f].getFootprint().setSpans(templateSpans[f]) 

1118 if self.config.conserveFlux: 

1119 fluxParents[f].set(self.nChildKey, nchild) 

1120 fluxParents[f].getFootprint().setSpans(fluxSpans[f]) 

1121 

1122 self.postSingleDeblendHook(exposure, fluxCatalogs, templateCatalogs, 

1123 pk, npre, foot, psfs, psf_fwhms, sigmas, result) 

1124 

1125 if fluxCatalogs is not None: 

1126 n1 = len(list(fluxCatalogs.values())[0]) 

1127 else: 

1128 n1 = len(list(templateCatalogs.values())[0]) 

1129 self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' 

1130 % (n0, nparents, n1-n0, n1)) 

1131 return fluxCatalogs, templateCatalogs 

1132 

1133 def preSingleDeblendHook(self, exposures, sources, pk, fp, psfs, psf_fwhms, sigmas): 

1134 pass 

1135 

1136 def postSingleDeblendHook(self, exposures, fluxCatalogs, templateCatalogs, 

1137 pk, npre, fp, psfs, psf_fwhms, sigmas, result): 

1138 pass 

1139 

1140 def isLargeFootprint(self, footprint): 

1141 """Returns whether a Footprint is large 

1142 

1143 'Large' is defined by thresholds on the area, size and axis ratio. 

1144 These may be disabled independently by configuring them to be non-positive. 

1145 

1146 This is principally intended to get rid of satellite streaks, which the 

1147 deblender or other downstream processing can have trouble dealing with 

1148 (e.g., multiple large HeavyFootprints can chew up memory). 

1149 """ 

1150 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

1151 return True 

1152 if self.config.maxFootprintSize > 0: 

1153 bbox = footprint.getBBox() 

1154 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

1155 return True 

1156 if self.config.minFootprintAxisRatio > 0: 

1157 axes = afwEll.Axes(footprint.getShape()) 

1158 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

1159 return True 

1160 return False 

1161 

1162 def isMasked(self, footprint, mask): 

1163 """Returns whether the footprint violates the mask limits""" 

1164 size = float(footprint.getArea()) 

1165 for maskName, limit in self.config.maskLimits.items(): 

1166 maskVal = mask.getPlaneBitMask(maskName) 

1167 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels 

1168 if (size - unmaskedSpan.getArea())/size > limit: 

1169 return True 

1170 return False 

1171 

1172 def skipParent(self, source, masks): 

1173 """Indicate that the parent source is not being deblended 

1174 

1175 We set the appropriate flags and masks for each exposure. 

1176 

1177 Parameters 

1178 ---------- 

1179 source: `lsst.afw.table.source.source.SourceRecord` 

1180 The source to flag as skipped 

1181 masks: list of `lsst.afw.image.MaskX` 

1182 The mask in each band to update with the non-detection 

1183 """ 

1184 fp = source.getFootprint() 

1185 source.set(self.deblendSkippedKey, True) 

1186 source.set(self.nChildKey, len(fp.getPeaks())) # It would have this many if we deblended them all 

1187 if self.config.notDeblendedMask: 

1188 for mask in masks: 

1189 mask.addMaskPlane(self.config.notDeblendedMask) 

1190 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))