Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_deblender. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy as np 

24 

25import lsst.log 

26import lsst.pex.config as pexConfig 

27import lsst.pipe.base as pipeBase 

28import lsst.afw.math as afwMath 

29import lsst.geom as geom 

30import lsst.afw.geom.ellipses as afwEll 

31import lsst.afw.image as afwImage 

32import lsst.afw.detection as afwDet 

33import lsst.afw.table as afwTable 

34 

35logger = lsst.log.Log.getLogger("meas.deblender.deblend") 

36 

37__all__ = 'SourceDeblendConfig', 'SourceDeblendTask' 

38 

39 

40class SourceDeblendConfig(pexConfig.Config): 

41 

42 edgeHandling = pexConfig.ChoiceField( 

43 doc='What to do when a peak to be deblended is close to the edge of the image', 

44 dtype=str, default='ramp', 

45 allowed={ 

46 'clip': 'Clip the template at the edge AND the mirror of the edge.', 

47 'ramp': 'Ramp down flux at the image edge by the PSF', 

48 'noclip': 'Ignore the edge when building the symmetric template.', 

49 } 

50 ) 

51 

52 strayFluxToPointSources = pexConfig.ChoiceField( 

53 doc='When the deblender should attribute stray flux to point sources', 

54 dtype=str, default='necessary', 

55 allowed={ 

56 'necessary': 'When there is not an extended object in the footprint', 

57 'always': 'Always', 

58 'never': ('Never; stray flux will not be attributed to any deblended child ' 

59 'if the deblender thinks all peaks look like point sources'), 

60 } 

61 ) 

62 

63 assignStrayFlux = pexConfig.Field(dtype=bool, default=True, 

64 doc='Assign stray flux (not claimed by any child in the deblender) ' 

65 'to deblend children.') 

66 

67 strayFluxRule = pexConfig.ChoiceField( 

68 doc='How to split flux among peaks', 

69 dtype=str, default='trim', 

70 allowed={ 

71 'r-to-peak': '~ 1/(1+R^2) to the peak', 

72 'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. ' 

73 'CAUTION: this can be computationally expensive on large footprints!'), 

74 'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka ' 

75 'Manhattan distance)'), 

76 'trim': ('Shrink the parent footprint to pixels that are not assigned to children') 

77 } 

78 ) 

79 

80 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001, 

81 doc=('When splitting stray flux, clip fractions below ' 

82 'this value to zero.')) 

83 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=False, 

84 doc=('Chi-squared per DOF cut for deciding a source is ' 

85 'a PSF during deblending (un-shifted PSF model)')) 

86 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=False, 

87 doc=('Chi-squared per DOF cut for deciding a source is ' 

88 'PSF during deblending (shifted PSF model)')) 

89 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=False, 

90 doc=('Chi-squared per DOF cut for deciding a source is ' 

91 'a PSF during deblending (shifted PSF model #2)')) 

92 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0, 

93 doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent" 

94 " (<= 0: unlimited)")) 

95 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000, 

96 doc=("Maximum area for footprints before they are ignored as large; " 

97 "non-positive means no threshold applied")) 

98 maxFootprintSize = pexConfig.Field(dtype=int, default=0, 

99 doc=("Maximum linear dimension for footprints before they are ignored " 

100 "as large; non-positive means no threshold applied")) 

101 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0, 

102 doc=("Minimum axis ratio for footprints before they are ignored " 

103 "as large; non-positive means no threshold applied")) 

104 notDeblendedMask = pexConfig.Field(dtype=str, default="NOT_DEBLENDED", optional=True, 

105 doc="Mask name for footprints not deblended, or None") 

106 

107 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=True, 

108 doc=('Footprints smaller in width or height than this value ' 

109 'will be ignored; minimum of 2 due to PSF gradient ' 

110 'calculation.')) 

111 

112 propagateAllPeaks = pexConfig.Field(dtype=bool, default=False, 

113 doc=('Guarantee that all peaks produce a child source.')) 

114 catchFailures = pexConfig.Field( 

115 dtype=bool, default=False, 

116 doc=("If True, catch exceptions thrown by the deblender, log them, " 

117 "and set a flag on the parent, instead of letting them propagate up")) 

118 maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"], 

119 doc="Mask planes to ignore when performing statistics") 

120 maskLimits = pexConfig.DictField( 

121 keytype=str, 

122 itemtype=float, 

123 default={}, 

124 doc=("Mask planes with the corresponding limit on the fraction of masked pixels. " 

125 "Sources violating this limit will not be deblended."), 

126 ) 

127 weightTemplates = pexConfig.Field( 

128 dtype=bool, default=False, 

129 doc=("If true, a least-squares fit of the templates will be done to the " 

130 "full image. The templates will be re-weighted based on this fit.")) 

131 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=False, 

132 doc=("Try to remove similar templates?")) 

133 maxTempDotProd = pexConfig.Field( 

134 dtype=float, default=0.5, 

135 doc=("If the dot product between two templates is larger than this value, we consider them to be " 

136 "describing the same object (i.e. they are degenerate). If one of the objects has been " 

137 "labeled as a PSF it will be removed, otherwise the template with the lowest value will " 

138 "be removed.")) 

139 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=True, 

140 doc="Apply a smoothing filter to all of the template images") 

141 

142## \addtogroup LSST_task_documentation 

143## \{ 

144## \page SourceDeblendTask 

145## \ref SourceDeblendTask_ "SourceDeblendTask" 

146## \copybrief SourceDeblendTask 

147## \} 

148 

149 

150class SourceDeblendTask(pipeBase.Task): 

151 """! 

152 \anchor SourceDeblendTask_ 

153 

154 \brief Split blended sources into individual sources. 

155 

156 This task has no return value; it only modifies the SourceCatalog in-place. 

157 """ 

158 ConfigClass = SourceDeblendConfig 

159 _DefaultName = "sourceDeblend" 

160 

161 def __init__(self, schema, peakSchema=None, **kwargs): 

162 """! 

163 Create the task, adding necessary fields to the given schema. 

164 

165 @param[in,out] schema Schema object for measurement fields; will be modified in-place. 

166 @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender. 

167 Any fields beyond the PeakTable minimal schema will be transferred 

168 to the main source Schema. If None, no fields will be transferred 

169 from the Peaks. 

170 @param[in] **kwargs Passed to Task.__init__. 

171 """ 

172 pipeBase.Task.__init__(self, **kwargs) 

173 self.schema = schema 

174 self.toCopyFromParent = [item.key for item in self.schema 

175 if item.field.getName().startswith("merge_footprint")] 

176 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() 

177 if peakSchema is None: 

178 # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one 

179 # to simplify downstream code 

180 self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) 

181 else: 

182 self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema) 

183 for item in peakSchema: 

184 if item.key not in peakMinimalSchema: 

185 self.peakSchemaMapper.addMapping(item.key, item.field) 

186 # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't 

187 # updating this Schema in place. That's probably a design flaw, but in the meantime, 

188 # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually, 

189 # by adding the same fields to both. 

190 schema.addField(item.field) 

191 assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas" 

192 self.addSchemaKeys(schema) 

193 

194 def addSchemaKeys(self, schema): 

195 self.nChildKey = schema.addField('deblend_nChild', type=np.int32, 

196 doc='Number of children this object has (defaults to 0)') 

197 self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag', 

198 doc='Deblender thought this source looked like a PSF') 

199 self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter', 

200 'If deblended-as-psf, the PSF centroid', "pixel") 

201 self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D', 

202 doc='If deblended-as-psf, the instrumental PSF flux', units='count') 

203 self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag', 

204 doc='Source had too many peaks; ' 

205 'only the brightest were included') 

206 self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag', 

207 doc='Parent footprint covered too many pixels') 

208 self.maskedKey = schema.addField('deblend_masked', type='Flag', 

209 doc='Parent footprint was predominantly masked') 

210 

211 if self.config.catchFailures: 

212 self.deblendFailedKey = schema.addField('deblend_failed', type='Flag', 

213 doc="Deblending failed on source") 

214 

215 self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag', 

216 doc="Deblender skipped this source") 

217 

218 self.deblendRampedTemplateKey = schema.addField( 

219 'deblend_rampedTemplate', type='Flag', 

220 doc=('This source was near an image edge and the deblender used ' 

221 '"ramp" edge-handling.')) 

222 

223 self.deblendPatchedTemplateKey = schema.addField( 

224 'deblend_patchedTemplate', type='Flag', 

225 doc=('This source was near an image edge and the deblender used ' 

226 '"patched" edge-handling.')) 

227 

228 self.hasStrayFluxKey = schema.addField( 

229 'deblend_hasStrayFlux', type='Flag', 

230 doc=('This source was assigned some stray flux')) 

231 

232 self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in ( 

233 self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey, 

234 self.tooManyPeaksKey, self.tooBigKey))) 

235 

236 @pipeBase.timeMethod 

237 def run(self, exposure, sources): 

238 """! 

239 Get the psf from the provided exposure and then run deblend(). 

240 

241 @param[in] exposure Exposure to process 

242 @param[in,out] sources SourceCatalog containing sources detected on this exposure. 

243 

244 @return None 

245 """ 

246 psf = exposure.getPsf() 

247 assert sources.getSchema() == self.schema 

248 self.deblend(exposure, sources, psf) 

249 

250 def _getPsfFwhm(self, psf, bbox): 

251 # It should be easier to get a PSF's fwhm; 

252 # https://dev.lsstcorp.org/trac/ticket/3030 

253 return psf.computeShape().getDeterminantRadius() * 2.35 

254 

255 @pipeBase.timeMethod 

256 def deblend(self, exposure, srcs, psf): 

257 """! 

258 Deblend. 

259 

260 @param[in] exposure Exposure to process 

261 @param[in,out] srcs SourceCatalog containing sources detected on this exposure. 

262 @param[in] psf PSF 

263 

264 @return None 

265 """ 

266 self.log.info("Deblending %d sources" % len(srcs)) 

267 

268 from lsst.meas.deblender.baseline import deblend 

269 

270 # find the median stdev in the image... 

271 mi = exposure.getMaskedImage() 

272 statsCtrl = afwMath.StatisticsControl() 

273 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes)) 

274 stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl) 

275 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN)) 

276 self.log.trace('sigma1: %g', sigma1) 

277 

278 n0 = len(srcs) 

279 nparents = 0 

280 for i, src in enumerate(srcs): 

281 # t0 = time.clock() 

282 

283 fp = src.getFootprint() 

284 pks = fp.getPeaks() 

285 

286 # Since we use the first peak for the parent object, we should propagate its flags 

287 # to the parent source. 

288 src.assign(pks[0], self.peakSchemaMapper) 

289 

290 if len(pks) < 2: 

291 continue 

292 

293 if self.isLargeFootprint(fp): 

294 src.set(self.tooBigKey, True) 

295 self.skipParent(src, mi.getMask()) 

296 self.log.warn('Parent %i: skipping large footprint (area: %i)', 

297 int(src.getId()), int(fp.getArea())) 

298 continue 

299 if self.isMasked(fp, exposure.getMaskedImage().getMask()): 

300 src.set(self.maskedKey, True) 

301 self.skipParent(src, mi.getMask()) 

302 self.log.warn('Parent %i: skipping masked footprint (area: %i)', 

303 int(src.getId()), int(fp.getArea())) 

304 continue 

305 

306 nparents += 1 

307 bb = fp.getBBox() 

308 psf_fwhm = self._getPsfFwhm(psf, bb) 

309 

310 self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks)) 

311 

312 self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1) 

313 npre = len(srcs) 

314 

315 # This should really be set in deblend, but deblend doesn't have access to the src 

316 src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks) 

317 

318 try: 

319 res = deblend( 

320 fp, mi, psf, psf_fwhm, sigma1=sigma1, 

321 psfChisqCut1=self.config.psfChisq1, 

322 psfChisqCut2=self.config.psfChisq2, 

323 psfChisqCut2b=self.config.psfChisq2b, 

324 maxNumberOfPeaks=self.config.maxNumberOfPeaks, 

325 strayFluxToPointSources=self.config.strayFluxToPointSources, 

326 assignStrayFlux=self.config.assignStrayFlux, 

327 strayFluxAssignment=self.config.strayFluxRule, 

328 rampFluxAtEdge=(self.config.edgeHandling == 'ramp'), 

329 patchEdges=(self.config.edgeHandling == 'noclip'), 

330 tinyFootprintSize=self.config.tinyFootprintSize, 

331 clipStrayFluxFraction=self.config.clipStrayFluxFraction, 

332 weightTemplates=self.config.weightTemplates, 

333 removeDegenerateTemplates=self.config.removeDegenerateTemplates, 

334 maxTempDotProd=self.config.maxTempDotProd, 

335 medianSmoothTemplate=self.config.medianSmoothTemplate 

336 ) 

337 if self.config.catchFailures: 

338 src.set(self.deblendFailedKey, False) 

339 except Exception as e: 

340 if self.config.catchFailures: 

341 self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e)) 

342 src.set(self.deblendFailedKey, True) 

343 import traceback 

344 traceback.print_exc() 

345 continue 

346 else: 

347 raise 

348 

349 kids = [] 

350 nchild = 0 

351 for j, peak in enumerate(res.deblendedParents[0].peaks): 

352 heavy = peak.getFluxPortion() 

353 if heavy is None or peak.skip: 

354 src.set(self.deblendSkippedKey, True) 

355 if not self.config.propagateAllPeaks: 

356 # Don't care 

357 continue 

358 # We need to preserve the peak: make sure we have enough info to create a minimal 

359 # child src 

360 self.log.trace("Peak at (%i,%i) failed. Using minimal default info for child.", 

361 pks[j].getIx(), pks[j].getIy()) 

362 if heavy is None: 

363 # copy the full footprint and strip out extra peaks 

364 foot = afwDet.Footprint(src.getFootprint()) 

365 peakList = foot.getPeaks() 

366 peakList.clear() 

367 peakList.append(peak.peak) 

368 zeroMimg = afwImage.MaskedImageF(foot.getBBox()) 

369 heavy = afwDet.makeHeavyFootprint(foot, zeroMimg) 

370 if peak.deblendedAsPsf: 

371 if peak.psfFitFlux is None: 

372 peak.psfFitFlux = 0.0 

373 if peak.psfFitCenter is None: 

374 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy()) 

375 

376 assert(len(heavy.getPeaks()) == 1) 

377 

378 src.set(self.deblendSkippedKey, False) 

379 child = srcs.addNew() 

380 nchild += 1 

381 for key in self.toCopyFromParent: 

382 child.set(key, src.get(key)) 

383 child.assign(heavy.getPeaks()[0], self.peakSchemaMapper) 

384 child.setParent(src.getId()) 

385 child.setFootprint(heavy) 

386 child.set(self.psfKey, peak.deblendedAsPsf) 

387 child.set(self.hasStrayFluxKey, peak.strayFlux is not None) 

388 if peak.deblendedAsPsf: 

389 (cx, cy) = peak.psfFitCenter 

390 child.set(self.psfCenterKey, geom.Point2D(cx, cy)) 

391 child.set(self.psfFluxKey, peak.psfFitFlux) 

392 child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate) 

393 child.set(self.deblendPatchedTemplateKey, peak.patched) 

394 kids.append(child) 

395 

396 # Child footprints may extend beyond the full extent of their parent's which 

397 # results in a failure of the replace-by-noise code to reinstate these pixels 

398 # to their original values. The following updates the parent footprint 

399 # in-place to ensure it contains the full union of itself and all of its 

400 # children's footprints. 

401 spans = src.getFootprint().spans 

402 for child in kids: 

403 spans = spans.union(child.getFootprint().spans) 

404 src.getFootprint().setSpans(spans) 

405 

406 src.set(self.nChildKey, nchild) 

407 

408 self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res) 

409 # print('Deblending parent id', src.getId(), 'took', time.clock() - t0) 

410 

411 n1 = len(srcs) 

412 self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' 

413 % (n0, nparents, n1-n0, n1)) 

414 

415 def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1): 

416 pass 

417 

418 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res): 

419 pass 

420 

421 def isLargeFootprint(self, footprint): 

422 """Returns whether a Footprint is large 

423 

424 'Large' is defined by thresholds on the area, size and axis ratio. 

425 These may be disabled independently by configuring them to be non-positive. 

426 

427 This is principally intended to get rid of satellite streaks, which the 

428 deblender or other downstream processing can have trouble dealing with 

429 (e.g., multiple large HeavyFootprints can chew up memory). 

430 """ 

431 if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea: 

432 return True 

433 if self.config.maxFootprintSize > 0: 

434 bbox = footprint.getBBox() 

435 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize: 

436 return True 

437 if self.config.minFootprintAxisRatio > 0: 

438 axes = afwEll.Axes(footprint.getShape()) 

439 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA(): 

440 return True 

441 return False 

442 

443 def isMasked(self, footprint, mask): 

444 """Returns whether the footprint violates the mask limits""" 

445 size = float(footprint.getArea()) 

446 for maskName, limit in self.config.maskLimits.items(): 

447 maskVal = mask.getPlaneBitMask(maskName) 

448 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels 

449 if (size - unmaskedSpan.getArea())/size > limit: 

450 return True 

451 return False 

452 

453 def skipParent(self, source, mask): 

454 """Indicate that the parent source is not being deblended 

455 

456 We set the appropriate flags and mask. 

457 

458 @param source The source to flag as skipped 

459 @param mask The mask to update 

460 """ 

461 fp = source.getFootprint() 

462 source.set(self.deblendSkippedKey, True) 

463 source.set(self.nChildKey, len(fp.getPeaks())) # It would have this many if we deblended them all 

464 if self.config.notDeblendedMask: 

465 mask.addMaskPlane(self.config.notDeblendedMask) 

466 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))