Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_deblender. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy as np 

24 

25import lsst.log 

26import lsst.pex.config as pexConfig 

27import lsst.pipe.base as pipeBase 

28import lsst.afw.math as afwMath 

29import lsst.geom as geom 

30import lsst.afw.geom.ellipses as afwEll 

31import lsst.afw.image as afwImage 

32import lsst.afw.detection as afwDet 

33import lsst.afw.table as afwTable 

34 

35logger = lsst.log.Log.getLogger("meas.deblender.deblend") 

36 

37__all__ = 'SourceDeblendConfig', 'SourceDeblendTask' 

38 

39 

40class SourceDeblendConfig(pexConfig.Config): 

41 

42 edgeHandling = pexConfig.ChoiceField( 

43 doc='What to do when a peak to be deblended is close to the edge of the image', 

44 dtype=str, default='ramp', 

45 allowed={ 

46 'clip': 'Clip the template at the edge AND the mirror of the edge.', 

47 'ramp': 'Ramp down flux at the image edge by the PSF', 

48 'noclip': 'Ignore the edge when building the symmetric template.', 

49 } 

50 ) 

51 

52 strayFluxToPointSources = pexConfig.ChoiceField( 

53 doc='When the deblender should attribute stray flux to point sources', 

54 dtype=str, default='necessary', 

55 allowed={ 

56 'necessary': 'When there is not an extended object in the footprint', 

57 'always': 'Always', 

58 'never': ('Never; stray flux will not be attributed to any deblended child ' 

59 'if the deblender thinks all peaks look like point sources'), 

60 } 

61 ) 

62 

63 assignStrayFlux = pexConfig.Field(dtype=bool, default=True, 

64 doc='Assign stray flux (not claimed by any child in the deblender) ' 

65 'to deblend children.') 

66 

67 strayFluxRule = pexConfig.ChoiceField( 

68 doc='How to split flux among peaks', 

69 dtype=str, default='trim', 

70 allowed={ 

71 'r-to-peak': '~ 1/(1+R^2) to the peak', 

72 'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. ' 

73 'CAUTION: this can be computationally expensive on large footprints!'), 

74 'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka ' 

75 'Manhattan distance)'), 

76 'trim': ('Shrink the parent footprint to pixels that are not assigned to children') 

77 } 

78 ) 

79 

80 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001, 

81 doc=('When splitting stray flux, clip fractions below ' 

82 'this value to zero.')) 

83 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=False, 

84 doc=('Chi-squared per DOF cut for deciding a source is ' 

85 'a PSF during deblending (un-shifted PSF model)')) 

86 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=False, 

87 doc=('Chi-squared per DOF cut for deciding a source is ' 

88 'PSF during deblending (shifted PSF model)')) 

89 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=False, 

90 doc=('Chi-squared per DOF cut for deciding a source is ' 

91 'a PSF during deblending (shifted PSF model #2)')) 

92 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0, 

93 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

94 " (<= 0: unlimited)")) 

95 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000, 

96 doc=("Maximum area for footprints before they are ignored as large; " 

97 "non-positive means no threshold applied")) 

98 maxFootprintSize = pexConfig.Field(dtype=int, default=0, 

99 doc=("Maximum linear dimension for footprints before they are ignored " 

100 "as large; non-positive means no threshold applied")) 

101 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0, 

102 doc=("Minimum axis ratio for footprints before they are ignored " 

103 "as large; non-positive means no threshold applied")) 

104 notDeblendedMask = pexConfig.Field(dtype=str, default="NOT_DEBLENDED", optional=True, 

105 doc="Mask name for footprints not deblended, or None") 

106 

107 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=True, 

108 doc=('Footprints smaller in width or height than this value ' 

109 'will be ignored; minimum of 2 due to PSF gradient ' 

110 'calculation.')) 

111 

112 propagateAllPeaks = pexConfig.Field(dtype=bool, default=False, 

113 doc=('Guarantee that all peaks produce a child source.')) 

114 catchFailures = pexConfig.Field( 

115 dtype=bool, default=False, 

116 doc=("If True, catch exceptions thrown by the deblender, log them, " 

117 "and set a flag on the parent, instead of letting them propagate up")) 

118 maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

119 doc="Mask planes to ignore when performing statistics") 

120 maskLimits = pexConfig.DictField( 

121 keytype=str, 

122 itemtype=float, 

123 default={}, 

124 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

125 "Sources violating this limit will not be deblended."), 

126 ) 

127 weightTemplates = pexConfig.Field( 

128 dtype=bool, default=False, 

129 doc=("If true, a least-squares fit of the templates will be done to the " 

130 "full image. The templates will be re-weighted based on this fit.")) 

131 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=False, 

132 doc=("Try to remove similar templates?")) 

133 maxTempDotProd = pexConfig.Field( 

134 dtype=float, default=0.5, 

135 doc=("If the dot product between two templates is larger than this value, we consider them to be " 

136 "describing the same object (i.e. they are degenerate). If one of the objects has been " 

137 "labeled as a PSF it will be removed, otherwise the template with the lowest value will " 

138 "be removed.")) 

139 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=True, 

140 doc="Apply a smoothing filter to all of the template images") 

141 

142## \addtogroup LSST_task_documentation 

143## \{ 

144## \page SourceDeblendTask 

145## \ref SourceDeblendTask_ "SourceDeblendTask" 

146## \copybrief SourceDeblendTask 

147## \} 

148 

149 

150class SourceDeblendTask(pipeBase.Task): 

151 """! 

152 \anchor SourceDeblendTask_ 

153 

154 \brief Split blended sources into individual sources. 

155 

156 This task has no return value; it only modifies the SourceCatalog in-place. 

157 """ 

158 ConfigClass = SourceDeblendConfig 

159 _DefaultName = "sourceDeblend" 

160 

161 def __init__(self, schema, peakSchema=None, **kwargs): 

162 """! 

163 Create the task, adding necessary fields to the given schema. 

164 

165 @param[in,out] schema Schema object for measurement fields; will be modified in-place. 

166 @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender. 

167 Any fields beyond the PeakTable minimal schema will be transferred 

168 to the main source Schema. If None, no fields will be transferred 

169 from the Peaks. 

170 @param[in] **kwargs Passed to Task.__init__. 

171 """ 

172 pipeBase.Task.__init__(self, **kwargs) 

173 self.schema = schema 

174 self.toCopyFromParent = [item.key for item in self.schema 

175 if item.field.getName().startswith("merge_footprint")] 

176 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

177 if peakSchema is None: 

178 # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one 

179 # to simplify downstream code 

180 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

181 else: 

182 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

183 for item in peakSchema: 

184 if item.key not in peakMinimalSchema: 

185 self.peakSchemaMapper.addMapping(item.key, item.field) 

186 # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't 

187 # updating this Schema in place. That's probably a design flaw, but in the meantime, 

188 # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually, 

189 # by adding the same fields to both. 

190 schema.addField(item.field) 

191 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

192 self.addSchemaKeys(schema) 

193 

194 def addSchemaKeys(self, schema): 

195 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

196 doc='Number of children this object has (defaults to 0)') 

197 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

198 doc='Deblender thought this source looked like a PSF') 

199 self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter', 

200 'If deblended-as-psf, the PSF centroid', "pixel") 

201 self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D', 

202 doc='If deblended-as-psf, the instrumental PSF flux', units='count') 

203 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

204 doc='Source had too many peaks; ' 

205 'only the brightest were included') 

206 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

207 doc='Parent footprint covered too many pixels') 

208 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

209 doc='Parent footprint was predominantly masked') 

210 

211 if self.config.catchFailures: 

212 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

213 doc="Deblending failed on source") 

214 

215 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

216 doc="Deblender skipped this source") 

217 

218 self.deblendRampedTemplateKey = schema.addField( 

219 'deblend_rampedTemplate', type='Flag', 

220 doc=('This source was near an image edge and the deblender used ' 

221 '"ramp" edge-handling.')) 

222 

223 self.deblendPatchedTemplateKey = schema.addField( 

224 'deblend_patchedTemplate', type='Flag', 

225 doc=('This source was near an image edge and the deblender used ' 

226 '"patched" edge-handling.')) 

227 

228 self.hasStrayFluxKey = schema.addField( 

229 'deblend_hasStrayFlux', type='Flag', 

230 doc=('This source was assigned some stray flux')) 

231 

232 self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in ( 

233 self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey, 

234 self.tooManyPeaksKey, self.tooBigKey))) 

235 self.peakCenter = afwTable.Point2IKey.addFields(schema, name="deblend_peak_center", 

236 doc="Center used to apply constraints in scarlet", 

237 unit="pixel") 

238 self.peakIdKey = schema.addField("deblend_peakId", type=np.int32, 

239 doc="ID of the peak in the parent footprint. " 

240 "This is not unique, but the combination of 'parent'" 

241 "and 'peakId' should be for all child sources. " 

242 "Top level blends with no parents have 'peakId=0'") 

243 self.nPeaksKey = schema.addField("deblend_nPeaks", type=np.int32, 

244 doc="Number of initial peaks in the blend. " 

245 "This includes peaks that may have been culled " 

246 "during deblending or failed to deblend") 

247 self.parentNPeaksKey = schema.addField("deblend_parentNPeaks", type=np.int32, 

248 doc="Same as deblend_n_peaks, but the number of peaks " 

249 "in the parent footprint") 

250 

251 @pipeBase.timeMethod 

252 def run(self, exposure, sources): 

253 """! 

254 Get the psf from the provided exposure and then run deblend(). 

255 

256 @param[in] exposure Exposure to process 

257 @param[in,out] sources SourceCatalog containing sources detected on this exposure. 

258 

259 @return None 

260 """ 

261 psf = exposure.getPsf() 

262 assert sources.getSchema() == self.schema 

263 self.deblend(exposure, sources, psf) 

264 

265 def _getPsfFwhm(self, psf, bbox): 

266 # It should be easier to get a PSF's fwhm; 

267 # https://dev.lsstcorp.org/trac/ticket/3030 

268 return psf.computeShape().getDeterminantRadius() * 2.35 

269 

270 @pipeBase.timeMethod 

271 def deblend(self, exposure, srcs, psf): 

272 """! 

273 Deblend. 

274 

275 @param[in] exposure Exposure to process 

276 @param[in,out] srcs SourceCatalog containing sources detected on this exposure. 

277 @param[in] psf PSF 

278 

279 @return None 

280 """ 

281 self.log.info("Deblending %d sources" % len(srcs)) 

282 

283 from lsst.meas.deblender.baseline import deblend 

284 

285 # find the median stdev in the image... 

286 mi = exposure.getMaskedImage() 

287 statsCtrl = afwMath.StatisticsControl() 

288 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes)) 

289 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl) 

290 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN)) 

291 self.log.trace('sigma1: %g', sigma1) 

292 

293 n0 = len(srcs) 

294 nparents = 0 

295 for i, src in enumerate(srcs): 

296 # t0 = time.clock() 

297 

298 fp = src.getFootprint() 

299 pks = fp.getPeaks() 

300 

301 # Since we use the first peak for the parent object, we should propagate its flags 

302 # to the parent source. 

303 src.assign(pks[0], self.peakSchemaMapper) 

304 

305 if len(pks) < 2: 

306 continue 

307 

308 if self.isLargeFootprint(fp): 

309 src.set(self.tooBigKey, True) 

310 self.skipParent(src, mi.getMask()) 

311 self.log.warn('Parent %i: skipping large footprint (area: %i)', 

312 int(src.getId()), int(fp.getArea())) 

313 continue 

314 if self.isMasked(fp, exposure.getMaskedImage().getMask()): 

315 src.set(self.maskedKey, True) 

316 self.skipParent(src, mi.getMask()) 

317 self.log.warn('Parent %i: skipping masked footprint (area: %i)', 

318 int(src.getId()), int(fp.getArea())) 

319 continue 

320 

321 nparents += 1 

322 bb = fp.getBBox() 

323 psf_fwhm = self._getPsfFwhm(psf, bb) 

324 

325 self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks)) 

326 

327 self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1) 

328 npre = len(srcs) 

329 

330 # This should really be set in deblend, but deblend doesn't have access to the src 

331 src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks) 

332 

333 try: 

334 res = deblend( 

335 fp, mi, psf, psf_fwhm, sigma1=sigma1, 

336 psfChisqCut1=self.config.psfChisq1, 

337 psfChisqCut2=self.config.psfChisq2, 

338 psfChisqCut2b=self.config.psfChisq2b, 

339 maxNumberOfPeaks=self.config.maxNumberOfPeaks, 

340 strayFluxToPointSources=self.config.strayFluxToPointSources, 

341 assignStrayFlux=self.config.assignStrayFlux, 

342 strayFluxAssignment=self.config.strayFluxRule, 

343 rampFluxAtEdge=(self.config.edgeHandling == 'ramp'), 

344 patchEdges=(self.config.edgeHandling == 'noclip'), 

345 tinyFootprintSize=self.config.tinyFootprintSize, 

346 clipStrayFluxFraction=self.config.clipStrayFluxFraction, 

347 weightTemplates=self.config.weightTemplates, 

348 removeDegenerateTemplates=self.config.removeDegenerateTemplates, 

349 maxTempDotProd=self.config.maxTempDotProd, 

350 medianSmoothTemplate=self.config.medianSmoothTemplate 

351 ) 

352 if self.config.catchFailures: 

353 src.set(self.deblendFailedKey, False) 

354 except Exception as e: 

355 if self.config.catchFailures: 

356 self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e)) 

357 src.set(self.deblendFailedKey, True) 

358 import traceback 

359 traceback.print_exc() 

360 continue 

361 else: 

362 raise 

363 

364 kids = [] 

365 nchild = 0 

366 for j, peak in enumerate(res.deblendedParents[0].peaks): 

367 heavy = peak.getFluxPortion() 

368 if heavy is None or peak.skip: 

369 src.set(self.deblendSkippedKey, True) 

370 if not self.config.propagateAllPeaks: 

371 # Don't care 

372 continue 

373 # We need to preserve the peak: make sure we have enough info to create a minimal 

374 # child src 

375 self.log.trace("Peak at (%i,%i) failed. Using minimal default info for child.", 

376 pks[j].getIx(), pks[j].getIy()) 

377 if heavy is None: 

378 # copy the full footprint and strip out extra peaks 

379 foot = afwDet.Footprint(src.getFootprint()) 

380 peakList = foot.getPeaks() 

381 peakList.clear() 

382 peakList.append(peak.peak) 

383 zeroMimg = afwImage.MaskedImageF(foot.getBBox()) 

384 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg) 

385 if peak.deblendedAsPsf: 

386 if peak.psfFitFlux is None: 

387 peak.psfFitFlux = 0.0 

388 if peak.psfFitCenter is None: 

389 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy()) 

390 

391 assert(len(heavy.getPeaks()) == 1) 

392 

393 src.set(self.deblendSkippedKey, False) 

394 child = srcs.addNew() 

395 nchild += 1 

396 for key in self.toCopyFromParent: 

397 child.set(key, src.get(key)) 

398 child.assign(heavy.getPeaks()[0], self.peakSchemaMapper) 

399 child.setParent(src.getId()) 

400 child.setFootprint(heavy) 

401 child.set(self.psfKey, peak.deblendedAsPsf) 

402 child.set(self.hasStrayFluxKey, peak.strayFlux is not None) 

403 if peak.deblendedAsPsf: 

404 (cx, cy) = peak.psfFitCenter 

405 child.set(self.psfCenterKey, geom.Point2D(cx, cy)) 

406 child.set(self.psfFluxKey, peak.psfFitFlux) 

407 child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate) 

408 child.set(self.deblendPatchedTemplateKey, peak.patched) 

409 

410 # Set the position of the peak from the parent footprint 

411 # This will make it easier to match the same source across 

412 # deblenders and across observations, where the peak 

413 # position is unlikely to change unless enough time passes 

414 # for a source to move on the sky. 

415 child.set(self.peakCenter, geom.Point2I(pks[j].getIx(), pks[j].getIy())) 

416 child.set(self.peakIdKey, pks[j].getId()) 

417 

418 # The children have a single peak 

419 child.set(self.nPeaksKey, 1) 

420 # Set the number of peaks in the parent 

421 child.set(self.parentNPeaksKey, len(pks)) 

422 

423 kids.append(child) 

424 

425 # Child footprints may extend beyond the full extent of their parent's which 

426 # results in a failure of the replace-by-noise code to reinstate these pixels 

427 # to their original values. The following updates the parent footprint 

428 # in-place to ensure it contains the full union of itself and all of its 

429 # children's footprints. 

430 spans = src.getFootprint().spans 

431 for child in kids: 

432 spans = spans.union(child.getFootprint().spans) 

433 src.getFootprint().setSpans(spans) 

434 

435 src.set(self.nChildKey, nchild) 

436 

437 self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res) 

438 # print('Deblending parent id', src.getId(), 'took', time.clock() - t0) 

439 

440 n1 = len(srcs) 

441 self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' 

442 % (n0, nparents, n1-n0, n1)) 

443 

444 def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1): 

445 pass 

446 

447 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res): 

448 pass 

449 

450 def isLargeFootprint(self, footprint): 

451 """Returns whether a Footprint is large 

452 

453 'Large' is defined by thresholds on the area, size and axis ratio. 

454 These may be disabled independently by configuring them to be non-positive. 

455 

456 This is principally intended to get rid of satellite streaks, which the 

457 deblender or other downstream processing can have trouble dealing with 

458 (e.g., multiple large HeavyFootprints can chew up memory). 

459 """ 

460 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

461 return True 

462 if self.config.maxFootprintSize > 0: 

463 bbox = footprint.getBBox() 

464 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

465 return True 

466 if self.config.minFootprintAxisRatio > 0: 

467 axes = afwEll.Axes(footprint.getShape()) 

468 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

469 return True 

470 return False 

471 

472 def isMasked(self, footprint, mask): 

473 """Returns whether the footprint violates the mask limits""" 

474 size = float(footprint.getArea()) 

475 for maskName, limit in self.config.maskLimits.items(): 

476 maskVal = mask.getPlaneBitMask(maskName) 

477 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels 

478 if (size - unmaskedSpan.getArea())/size > limit: 

479 return True 

480 return False 

481 

482 def skipParent(self, source, mask): 

483 """Indicate that the parent source is not being deblended 

484 

485 We set the appropriate flags and mask. 

486 

487 @param source The source to flag as skipped 

488 @param mask The mask to update 

489 """ 

490 fp = source.getFootprint() 

491 source.set(self.deblendSkippedKey, True) 

492 if self.config.notDeblendedMask: 

493 mask.addMaskPlane(self.config.notDeblendedMask) 

494 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask)) 

495 

496 # Set the center of the parent 

497 bbox = fp.getBBox() 

498 centerX = int(bbox.getMinX()+bbox.getWidth()/2) 

499 centerY = int(bbox.getMinY()+bbox.getHeight()/2) 

500 source.set(self.peakCenter, geom.Point2I(centerX, centerY)) 

501 # There are no deblended children, so nChild = 0 

502 source.set(self.nChildKey, 0) 

503 # But we also want to know how many peaks that we would have 

504 # deblended if the parent wasn't skipped. 

505 source.set(self.nPeaksKey, len(fp.peaks)) 

506 # Top level parents are not a detected peak, so they have no peakId 

507 source.set(self.peakIdKey, 0) 

508 # Top level parents also have no parentNPeaks 

509 source.set(self.parentNPeaksKey, 0)