Coverage for python/lsst/meas/algorithms/objectSizeStarSelector.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

249 statements  

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["ObjectSizeStarSelectorConfig", "ObjectSizeStarSelectorTask"] 

23 

24import sys 

25 

26import numpy 

27import warnings 

28from functools import reduce 

29 

30from lsst.utils.logging import getLogger 

31from lsst.pipe.base import Struct 

32import lsst.geom 

33from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS 

34import lsst.afw.geom as afwGeom 

35import lsst.pex.config as pexConfig 

36import lsst.afw.display as afwDisplay 

37from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry 

38 

39afwDisplay.setDefaultMaskTransparency(75) 

40 

41_LOG = getLogger(__name__) 

42 

43 

44class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass): 

45 doFluxLimit = pexConfig.Field( 

46 doc="Apply flux limit to Psf Candidate selection?", 

47 dtype=bool, 

48 default=True, 

49 ) 

50 fluxMin = pexConfig.Field( 50 ↛ exitline 50 didn't jump to the function exit

51 doc="specify the minimum psfFlux for good Psf Candidates", 

52 dtype=float, 

53 default=12500.0, 

54 check=lambda x: x >= 0.0, 

55 ) 

56 fluxMax = pexConfig.Field( 56 ↛ exitline 56 didn't jump to the function exit

57 doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)", 

58 dtype=float, 

59 default=0.0, 

60 check=lambda x: x >= 0.0, 

61 ) 

62 doSignalToNoiseLimit = pexConfig.Field( 

63 doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?", 

64 dtype=bool, 

65 default=False, 

66 ) 

67 signalToNoiseMin = pexConfig.Field( 67 ↛ exitline 67 didn't jump to the function exit

68 doc="specify the minimum signal-to-noise for good Psf Candidates", 

69 dtype=float, 

70 default=20.0, 

71 check=lambda x: x >= 0.0, 

72 ) 

73 signalToNoiseMax = pexConfig.Field( 73 ↛ exitline 73 didn't jump to the function exit

74 doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)", 

75 dtype=float, 

76 default=0.0, 

77 check=lambda x: x >= 0.0, 

78 ) 

79 widthMin = pexConfig.Field( 79 ↛ exitline 79 didn't jump to the function exit

80 doc="minimum width to include in histogram", 

81 dtype=float, 

82 default=0.0, 

83 check=lambda x: x >= 0.0, 

84 ) 

85 widthMax = pexConfig.Field( 85 ↛ exitline 85 didn't jump to the function exit

86 doc="maximum width to include in histogram", 

87 dtype=float, 

88 default=10.0, 

89 check=lambda x: x >= 0.0, 

90 ) 

91 sourceFluxField = pexConfig.Field( 

92 doc="Name of field in Source to use for flux measurement", 

93 dtype=str, 

94 default="base_GaussianFlux_instFlux", 

95 ) 

96 widthStdAllowed = pexConfig.Field( 96 ↛ exitline 96 didn't jump to the function exit

97 doc="Standard deviation of width allowed to be interpreted as good stars", 

98 dtype=float, 

99 default=0.15, 

100 check=lambda x: x >= 0.0, 

101 ) 

102 nSigmaClip = pexConfig.Field( 102 ↛ exitline 102 didn't jump to the function exit

103 doc="Keep objects within this many sigma of cluster 0's median", 

104 dtype=float, 

105 default=2.0, 

106 check=lambda x: x >= 0.0, 

107 ) 

108 badFlags = pexConfig.ListField( 

109 doc="List of flags which cause a source to be rejected as bad", 

110 dtype=str, 

111 default=[ 

112 "base_PixelFlags_flag_edge", 

113 "base_PixelFlags_flag_interpolatedCenter", 

114 "base_PixelFlags_flag_saturatedCenter", 

115 "base_PixelFlags_flag_crCenter", 

116 "base_PixelFlags_flag_bad", 

117 "base_PixelFlags_flag_interpolated", 

118 ], 

119 ) 

120 

121 def validate(self): 

122 BaseSourceSelectorTask.ConfigClass.validate(self) 

123 if self.widthMin > self.widthMax: 

124 msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})" 

125 raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg) 

126 

127 

128class EventHandler: 

129 """A class to handle key strokes with matplotlib displays. 

130 """ 

131 

132 def __init__(self, axes, xs, ys, x, y, frames=[0]): 

133 self.axes = axes 

134 self.xs = xs 

135 self.ys = ys 

136 self.x = x 

137 self.y = y 

138 self.frames = frames 

139 

140 self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self) 

141 

142 def __call__(self, ev): 

143 if ev.inaxes != self.axes: 

144 return 

145 

146 if ev.key and ev.key in ("p"): 

147 dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata) 

148 dist[numpy.where(numpy.isnan(dist))] = 1e30 

149 

150 which = numpy.where(dist == min(dist)) 

151 

152 x = self.x[which][0] 

153 y = self.y[which][0] 

154 for frame in self.frames: 

155 disp = afwDisplay.Display(frame=frame) 

156 disp.pan(x, y) 

157 disp.flush() 

158 else: 

159 pass 

160 

161 

162def _assignClusters(yvec, centers): 

163 """Return a vector of centerIds based on their distance to the centers. 

164 """ 

165 assert len(centers) > 0 

166 

167 minDist = numpy.nan*numpy.ones_like(yvec) 

168 clusterId = numpy.empty_like(yvec) 

169 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

170 dbl = _LOG.getChild("_assignClusters") 

171 dbl.setLevel(dbl.INFO) 

172 

173 # Make sure we are logging aall numpy warnings... 

174 oldSettings = numpy.seterr(all="warn") 

175 with warnings.catch_warnings(record=True) as w: 

176 warnings.simplefilter("always") 

177 for i, mean in enumerate(centers): 

178 dist = abs(yvec - mean) 

179 if i == 0: 

180 update = dist == dist # True for all points 

181 else: 

182 update = dist < minDist 

183 if w: # Only do if w is not empty i.e. contains a warning message 

184 dbl.trace(str(w[-1])) 

185 

186 minDist[update] = dist[update] 

187 clusterId[update] = i 

188 numpy.seterr(**oldSettings) 

189 

190 return clusterId 

191 

192 

193def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15): 

194 """A classic k-means algorithm, clustering yvec into nCluster clusters 

195 

196 Return the set of centres, and the cluster ID for each of the points 

197 

198 If useMedian is true, use the median of the cluster as its centre, rather than 

199 the traditional mean 

200 

201 Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means: 

202 "e.g. why not use the Forgy or random partition initialization methods" 

203 however, the approach adopted here seems to work well for the particular sorts of things 

204 we're clustering in this application 

205 """ 

206 

207 assert nCluster > 0 

208 

209 mean0 = sorted(yvec)[len(yvec)//10] # guess 

210 delta = mean0 * widthStdAllowed * 2.0 

211 centers = mean0 + delta * numpy.arange(nCluster) 

212 

213 func = numpy.median if useMedian else numpy.mean 

214 

215 clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to 

216 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

217 while True: 

218 oclusterId = clusterId 

219 clusterId = _assignClusters(yvec, centers) 

220 

221 if numpy.all(clusterId == oclusterId): 

222 break 

223 

224 for i in range(nCluster): 

225 # Only compute func if some points are available; otherwise, default to NaN. 

226 pointsInCluster = (clusterId == i) 

227 if numpy.any(pointsInCluster): 

228 centers[i] = func(yvec[pointsInCluster]) 

229 else: 

230 centers[i] = numpy.nan 

231 

232 return centers, clusterId 

233 

234 

235def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15): 

236 """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median. 

237 """ 

238 

239 nMember = sum(clusterId == clusterNum) 

240 if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement 

241 return clusterId 

242 for iter in range(nIteration): 

243 old_nMember = nMember 

244 

245 inCluster0 = clusterId == clusterNum 

246 yv = yvec[inCluster0] 

247 

248 centers[clusterNum] = numpy.median(yv) 

249 stdev = numpy.std(yv) 

250 

251 syv = sorted(yv) 

252 stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)]) 

253 median = syv[int(0.5*nMember)] 

254 

255 sd = stdev if stdev < stdev_iqr else stdev_iqr 

256 

257 if False: 

258 print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv))) 

259 newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd 

260 clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum 

261 clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1 

262 

263 nMember = sum(clusterId == clusterNum) 

264 # 'sd < widthStdAllowed * median' prevents too much rejections 

265 if nMember == old_nMember or sd < widthStdAllowed * median: 

266 break 

267 

268 return clusterId 

269 

270 

271def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', 

272 magType="model", clear=True): 

273 

274 log = _LOG.getChild("plot") 

275 try: 

276 import matplotlib.pyplot as plt 

277 except ImportError as e: 

278 log.warning("Unable to import matplotlib: %s", e) 

279 return 

280 

281 try: 

282 fig 

283 except NameError: 

284 fig = plt.figure() 

285 else: 

286 if clear: 

287 fig.clf() 

288 

289 axes = fig.add_axes((0.1, 0.1, 0.85, 0.80)) 

290 

291 xmin = sorted(mag)[int(0.05*len(mag))] 

292 xmax = sorted(mag)[int(0.95*len(mag))] 

293 

294 axes.set_xlim(-17.5, -13) 

295 axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin)) 

296 axes.set_ylim(0, 10) 

297 

298 colors = ["r", "g", "b", "c", "m", "k", ] 

299 for k, mean in enumerate(centers): 

300 if k == 0: 

301 axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype) 

302 

303 li = (clusterId == k) 

304 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

305 color=colors[k % len(colors)]) 

306 

307 li = (clusterId == -1) 

308 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

309 color='k') 

310 

311 if clear: 

312 axes.set_xlabel("Instrumental %s mag" % magType) 

313 axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$") 

314 

315 return fig 

316 

317 

318@pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry) 

319class ObjectSizeStarSelectorTask(BaseSourceSelectorTask): 

320 r"""A star selector that looks for a cluster of small objects in a size-magnitude plot. 

321 """ 

322 ConfigClass = ObjectSizeStarSelectorConfig 

323 usesMatches = False # selectStars does not use its matches argument 

324 

325 def selectSources(self, sourceCat, matches=None, exposure=None): 

326 """Return a selection of PSF candidates that represent likely stars. 

327 

328 A list of PSF candidates may be used by a PSF fitter to construct a PSF. 

329 

330 Parameters: 

331 ----------- 

332 sourceCat : `lsst.afw.table.SourceCatalog` 

333 Catalog of sources to select from. 

334 This catalog must be contiguous in memory. 

335 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

336 Ignored in this SourceSelector. 

337 exposure : `lsst.afw.image.Exposure` or None 

338 The exposure the catalog was built from; used to get the detector 

339 to transform to TanPix, and for debug display. 

340 

341 Return 

342 ------ 

343 struct : `lsst.pipe.base.Struct` 

344 The struct contains the following data: 

345 

346 - selected : `array` of `bool`` 

347 Boolean array of sources that were selected, same length as 

348 sourceCat. 

349 """ 

350 import lsstDebug 

351 display = lsstDebug.Info(__name__).display 

352 displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

353 plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation 

354 dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file? 

355 

356 detector = None 

357 pixToTanPix = None 

358 if exposure: 

359 detector = exposure.getDetector() 

360 if detector: 

361 pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS) 

362 # 

363 # Look at the distribution of stars in the magnitude-size plane 

364 # 

365 flux = sourceCat.get(self.config.sourceFluxField) 

366 fluxErr = sourceCat.get(self.config.sourceFluxField + "Err") 

367 

368 xx = numpy.empty(len(sourceCat)) 

369 xy = numpy.empty_like(xx) 

370 yy = numpy.empty_like(xx) 

371 for i, source in enumerate(sourceCat): 

372 Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy() 

373 if pixToTanPix: 

374 p = lsst.geom.Point2D(source.getX(), source.getY()) 

375 linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear() 

376 m = afwGeom.Quadrupole(Ixx, Iyy, Ixy) 

377 m.transform(linTransform) 

378 Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy() 

379 

380 xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy 

381 

382 width = numpy.sqrt(0.5*(xx + yy)) 

383 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

384 bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) 

385 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width))) 

386 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux))) 

387 if self.config.doFluxLimit: 

388 bad = numpy.logical_or(bad, flux < self.config.fluxMin) 

389 if self.config.fluxMax > 0: 

390 bad = numpy.logical_or(bad, flux > self.config.fluxMax) 

391 if self.config.doSignalToNoiseLimit: 

392 bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin) 

393 if self.config.signalToNoiseMax > 0: 

394 bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax) 

395 bad = numpy.logical_or(bad, width < self.config.widthMin) 

396 bad = numpy.logical_or(bad, width > self.config.widthMax) 

397 good = numpy.logical_not(bad) 

398 

399 if not numpy.any(good): 

400 raise RuntimeError("No objects passed our cuts for consideration as psf stars") 

401 

402 mag = -2.5*numpy.log10(flux[good]) 

403 width = width[good] 

404 # 

405 # Look for the maximum in the size histogram, then search upwards for the minimum that separates 

406 # the initial peak (of, we presume, stars) from the galaxies 

407 # 

408 if dumpData: 

409 import os 

410 import pickle as pickle 

411 _ii = 0 

412 while True: 

413 pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii)) 

414 if not os.path.exists(pickleFile): 

415 break 

416 _ii += 1 

417 

418 with open(pickleFile, "wb") as fd: 

419 pickle.dump(mag, fd, -1) 

420 pickle.dump(width, fd, -1) 

421 

422 centers, clusterId = _kcenters(width, nCluster=4, useMedian=True, 

423 widthStdAllowed=self.config.widthStdAllowed) 

424 

425 if display and plotMagSize: 

426 fig = plot(mag, width, centers, clusterId, 

427 magType=self.config.sourceFluxField.split(".")[-1].title(), 

428 marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True) 

429 else: 

430 fig = None 

431 

432 clusterId = _improveCluster(width, centers, clusterId, 

433 nsigma=self.config.nSigmaClip, 

434 widthStdAllowed=self.config.widthStdAllowed) 

435 

436 if display and plotMagSize: 

437 plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False) 

438 

439 stellar = (clusterId == 0) 

440 # 

441 # We know enough to plot, if so requested 

442 # 

443 frame = 0 

444 

445 if fig: 

446 if display and displayExposure: 

447 disp = afwDisplay.Display(frame=frame) 

448 disp.mtv(exposure.getMaskedImage(), title="PSF candidates") 

449 

450 global eventHandler 

451 eventHandler = EventHandler(fig.get_axes()[0], mag, width, 

452 sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame]) 

453 

454 fig.show() 

455 

456 while True: 

457 try: 

458 reply = input("continue? [c h(elp) q(uit) p(db)] ").strip() 

459 except EOFError: 

460 reply = None 

461 if not reply: 

462 reply = "c" 

463 

464 if reply: 

465 if reply[0] == "h": 

466 print("""\ 

467 We cluster the points; red are the stellar candidates and the other colours are other clusters. 

468 Points labelled + are rejects from the cluster (only for cluster 0). 

469 

470 At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text 

471 

472 If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the 

473 image display. 

474 """) 

475 elif reply[0] == "p": 

476 import pdb 

477 pdb.set_trace() 

478 elif reply[0] == 'q': 

479 sys.exit(1) 

480 else: 

481 break 

482 

483 if display and displayExposure: 

484 mi = exposure.getMaskedImage() 

485 with disp.Buffering(): 

486 for i, source in enumerate(sourceCat): 

487 if good[i]: 

488 ctype = afwDisplay.GREEN # star candidate 

489 else: 

490 ctype = afwDisplay.RED # not star 

491 

492 disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) 

493 

494 # stellar only applies to good==True objects 

495 mask = good == True # noqa (numpy bool comparison): E712 

496 good[mask] = stellar 

497 

498 return Struct(selected=good)