Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import sys 

23 

24import numpy 

25import warnings 

26from functools import reduce 

27 

28from lsst.log import Log 

29from lsst.pipe.base import Struct 

30import lsst.geom 

31from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS 

32import lsst.afw.geom as afwGeom 

33import lsst.pex.config as pexConfig 

34import lsst.afw.display as afwDisplay 

35from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry 

36 

37afwDisplay.setDefaultMaskTransparency(75) 

38 

39 

40class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass): 

41 doFluxLimit = pexConfig.Field( 

42 doc="Apply flux limit to Psf Candidate selection?", 

43 dtype=bool, 

44 default=True, 

45 ) 

46 fluxMin = pexConfig.Field( 46 ↛ exitline 46 didn't jump to the function exit

47 doc="specify the minimum psfFlux for good Psf Candidates", 

48 dtype=float, 

49 default=12500.0, 

50 check=lambda x: x >= 0.0, 

51 ) 

52 fluxMax = pexConfig.Field( 52 ↛ exitline 52 didn't jump to the function exit

53 doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)", 

54 dtype=float, 

55 default=0.0, 

56 check=lambda x: x >= 0.0, 

57 ) 

58 doSignalToNoiseLimit = pexConfig.Field( 

59 doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?", 

60 dtype=bool, 

61 default=False, 

62 ) 

63 signalToNoiseMin = pexConfig.Field( 63 ↛ exitline 63 didn't jump to the function exit

64 doc="specify the minimum signal-to-noise for good Psf Candidates", 

65 dtype=float, 

66 default=20.0, 

67 check=lambda x: x >= 0.0, 

68 ) 

69 signalToNoiseMax = pexConfig.Field( 69 ↛ exitline 69 didn't jump to the function exit

70 doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)", 

71 dtype=float, 

72 default=0.0, 

73 check=lambda x: x >= 0.0, 

74 ) 

75 widthMin = pexConfig.Field( 75 ↛ exitline 75 didn't jump to the function exit

76 doc="minimum width to include in histogram", 

77 dtype=float, 

78 default=0.0, 

79 check=lambda x: x >= 0.0, 

80 ) 

81 widthMax = pexConfig.Field( 81 ↛ exitline 81 didn't jump to the function exit

82 doc="maximum width to include in histogram", 

83 dtype=float, 

84 default=10.0, 

85 check=lambda x: x >= 0.0, 

86 ) 

87 sourceFluxField = pexConfig.Field( 

88 doc="Name of field in Source to use for flux measurement", 

89 dtype=str, 

90 default="base_GaussianFlux_instFlux", 

91 ) 

92 widthStdAllowed = pexConfig.Field( 92 ↛ exitline 92 didn't jump to the function exit

93 doc="Standard deviation of width allowed to be interpreted as good stars", 

94 dtype=float, 

95 default=0.15, 

96 check=lambda x: x >= 0.0, 

97 ) 

98 nSigmaClip = pexConfig.Field( 98 ↛ exitline 98 didn't jump to the function exit

99 doc="Keep objects within this many sigma of cluster 0's median", 

100 dtype=float, 

101 default=2.0, 

102 check=lambda x: x >= 0.0, 

103 ) 

104 badFlags = pexConfig.ListField( 

105 doc="List of flags which cause a source to be rejected as bad", 

106 dtype=str, 

107 default=[ 

108 "base_PixelFlags_flag_edge", 

109 "base_PixelFlags_flag_interpolatedCenter", 

110 "base_PixelFlags_flag_saturatedCenter", 

111 "base_PixelFlags_flag_crCenter", 

112 "base_PixelFlags_flag_bad", 

113 "base_PixelFlags_flag_interpolated", 

114 ], 

115 ) 

116 

117 def validate(self): 

118 BaseSourceSelectorTask.ConfigClass.validate(self) 

119 if self.widthMin > self.widthMax: 

120 msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})" 

121 raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg) 

122 

123 

124class EventHandler: 

125 """A class to handle key strokes with matplotlib displays. 

126 """ 

127 

128 def __init__(self, axes, xs, ys, x, y, frames=[0]): 

129 self.axes = axes 

130 self.xs = xs 

131 self.ys = ys 

132 self.x = x 

133 self.y = y 

134 self.frames = frames 

135 

136 self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self) 

137 

138 def __call__(self, ev): 

139 if ev.inaxes != self.axes: 

140 return 

141 

142 if ev.key and ev.key in ("p"): 

143 dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata) 

144 dist[numpy.where(numpy.isnan(dist))] = 1e30 

145 

146 which = numpy.where(dist == min(dist)) 

147 

148 x = self.x[which][0] 

149 y = self.y[which][0] 

150 for frame in self.frames: 

151 disp = afwDisplay.Display(frame=frame) 

152 disp.pan(x, y) 

153 disp.flush() 

154 else: 

155 pass 

156 

157 

158def _assignClusters(yvec, centers): 

159 """Return a vector of centerIds based on their distance to the centers. 

160 """ 

161 assert len(centers) > 0 

162 

163 minDist = numpy.nan*numpy.ones_like(yvec) 

164 clusterId = numpy.empty_like(yvec) 

165 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

166 dbl = Log.getLogger("objectSizeStarSelector._assignClusters") 

167 dbl.setLevel(dbl.INFO) 

168 

169 # Make sure we are logging aall numpy warnings... 

170 oldSettings = numpy.seterr(all="warn") 

171 with warnings.catch_warnings(record=True) as w: 

172 warnings.simplefilter("always") 

173 for i, mean in enumerate(centers): 

174 dist = abs(yvec - mean) 

175 if i == 0: 

176 update = dist == dist # True for all points 

177 else: 

178 update = dist < minDist 

179 if w: # Only do if w is not empty i.e. contains a warning message 

180 dbl.trace(str(w[-1])) 

181 

182 minDist[update] = dist[update] 

183 clusterId[update] = i 

184 numpy.seterr(**oldSettings) 

185 

186 return clusterId 

187 

188 

189def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15): 

190 """A classic k-means algorithm, clustering yvec into nCluster clusters 

191 

192 Return the set of centres, and the cluster ID for each of the points 

193 

194 If useMedian is true, use the median of the cluster as its centre, rather than 

195 the traditional mean 

196 

197 Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means: 

198 "e.g. why not use the Forgy or random partition initialization methods" 

199 however, the approach adopted here seems to work well for the particular sorts of things 

200 we're clustering in this application 

201 """ 

202 

203 assert nCluster > 0 

204 

205 mean0 = sorted(yvec)[len(yvec)//10] # guess 

206 delta = mean0 * widthStdAllowed * 2.0 

207 centers = mean0 + delta * numpy.arange(nCluster) 

208 

209 func = numpy.median if useMedian else numpy.mean 

210 

211 clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to 

212 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

213 while True: 

214 oclusterId = clusterId 

215 clusterId = _assignClusters(yvec, centers) 

216 

217 if numpy.all(clusterId == oclusterId): 

218 break 

219 

220 for i in range(nCluster): 

221 # Only compute func if some points are available; otherwise, default to NaN. 

222 pointsInCluster = (clusterId == i) 

223 if numpy.any(pointsInCluster): 

224 centers[i] = func(yvec[pointsInCluster]) 

225 else: 

226 centers[i] = numpy.nan 

227 

228 return centers, clusterId 

229 

230 

231def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15): 

232 """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median. 

233 """ 

234 

235 nMember = sum(clusterId == clusterNum) 

236 if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement 

237 return clusterId 

238 for iter in range(nIteration): 

239 old_nMember = nMember 

240 

241 inCluster0 = clusterId == clusterNum 

242 yv = yvec[inCluster0] 

243 

244 centers[clusterNum] = numpy.median(yv) 

245 stdev = numpy.std(yv) 

246 

247 syv = sorted(yv) 

248 stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)]) 

249 median = syv[int(0.5*nMember)] 

250 

251 sd = stdev if stdev < stdev_iqr else stdev_iqr 

252 

253 if False: 

254 print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv))) 

255 newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd 

256 clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum 

257 clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1 

258 

259 nMember = sum(clusterId == clusterNum) 

260 # 'sd < widthStdAllowed * median' prevents too much rejections 

261 if nMember == old_nMember or sd < widthStdAllowed * median: 

262 break 

263 

264 return clusterId 

265 

266 

267def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', 

268 magType="model", clear=True): 

269 

270 log = Log.getLogger("objectSizeStarSelector.plot") 

271 try: 

272 import matplotlib.pyplot as plt 

273 except ImportError as e: 

274 log.warning("Unable to import matplotlib: %s", e) 

275 return 

276 

277 try: 

278 fig 

279 except NameError: 

280 fig = plt.figure() 

281 else: 

282 if clear: 

283 fig.clf() 

284 

285 axes = fig.add_axes((0.1, 0.1, 0.85, 0.80)) 

286 

287 xmin = sorted(mag)[int(0.05*len(mag))] 

288 xmax = sorted(mag)[int(0.95*len(mag))] 

289 

290 axes.set_xlim(-17.5, -13) 

291 axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin)) 

292 axes.set_ylim(0, 10) 

293 

294 colors = ["r", "g", "b", "c", "m", "k", ] 

295 for k, mean in enumerate(centers): 

296 if k == 0: 

297 axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype) 

298 

299 li = (clusterId == k) 

300 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

301 color=colors[k % len(colors)]) 

302 

303 li = (clusterId == -1) 

304 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

305 color='k') 

306 

307 if clear: 

308 axes.set_xlabel("Instrumental %s mag" % magType) 

309 axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$") 

310 

311 return fig 

312 

313 

314@pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry) 

315class ObjectSizeStarSelectorTask(BaseSourceSelectorTask): 

316 r"""A star selector that looks for a cluster of small objects in a size-magnitude plot. 

317 """ 

318 ConfigClass = ObjectSizeStarSelectorConfig 

319 usesMatches = False # selectStars does not use its matches argument 

320 

321 def selectSources(self, sourceCat, matches=None, exposure=None): 

322 """Return a selection of PSF candidates that represent likely stars. 

323 

324 A list of PSF candidates may be used by a PSF fitter to construct a PSF. 

325 

326 Parameters: 

327 ----------- 

328 sourceCat : `lsst.afw.table.SourceCatalog` 

329 Catalog of sources to select from. 

330 This catalog must be contiguous in memory. 

331 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

332 Ignored in this SourceSelector. 

333 exposure : `lsst.afw.image.Exposure` or None 

334 The exposure the catalog was built from; used to get the detector 

335 to transform to TanPix, and for debug display. 

336 

337 Return 

338 ------ 

339 struct : `lsst.pipe.base.Struct` 

340 The struct contains the following data: 

341 

342 - selected : `array` of `bool`` 

343 Boolean array of sources that were selected, same length as 

344 sourceCat. 

345 """ 

346 import lsstDebug 

347 display = lsstDebug.Info(__name__).display 

348 displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

349 plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation 

350 dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file? 

351 

352 detector = None 

353 pixToTanPix = None 

354 if exposure: 

355 detector = exposure.getDetector() 

356 if detector: 

357 pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS) 

358 # 

359 # Look at the distribution of stars in the magnitude-size plane 

360 # 

361 flux = sourceCat.get(self.config.sourceFluxField) 

362 fluxErr = sourceCat.get(self.config.sourceFluxField + "Err") 

363 

364 xx = numpy.empty(len(sourceCat)) 

365 xy = numpy.empty_like(xx) 

366 yy = numpy.empty_like(xx) 

367 for i, source in enumerate(sourceCat): 

368 Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy() 

369 if pixToTanPix: 

370 p = lsst.geom.Point2D(source.getX(), source.getY()) 

371 linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear() 

372 m = afwGeom.Quadrupole(Ixx, Iyy, Ixy) 

373 m.transform(linTransform) 

374 Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy() 

375 

376 xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy 

377 

378 width = numpy.sqrt(0.5*(xx + yy)) 

379 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

380 bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) 

381 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width))) 

382 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux))) 

383 if self.config.doFluxLimit: 

384 bad = numpy.logical_or(bad, flux < self.config.fluxMin) 

385 if self.config.fluxMax > 0: 

386 bad = numpy.logical_or(bad, flux > self.config.fluxMax) 

387 if self.config.doSignalToNoiseLimit: 

388 bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin) 

389 if self.config.signalToNoiseMax > 0: 

390 bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax) 

391 bad = numpy.logical_or(bad, width < self.config.widthMin) 

392 bad = numpy.logical_or(bad, width > self.config.widthMax) 

393 good = numpy.logical_not(bad) 

394 

395 if not numpy.any(good): 

396 raise RuntimeError("No objects passed our cuts for consideration as psf stars") 

397 

398 mag = -2.5*numpy.log10(flux[good]) 

399 width = width[good] 

400 # 

401 # Look for the maximum in the size histogram, then search upwards for the minimum that separates 

402 # the initial peak (of, we presume, stars) from the galaxies 

403 # 

404 if dumpData: 

405 import os 

406 import pickle as pickle 

407 _ii = 0 

408 while True: 

409 pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii)) 

410 if not os.path.exists(pickleFile): 

411 break 

412 _ii += 1 

413 

414 with open(pickleFile, "wb") as fd: 

415 pickle.dump(mag, fd, -1) 

416 pickle.dump(width, fd, -1) 

417 

418 centers, clusterId = _kcenters(width, nCluster=4, useMedian=True, 

419 widthStdAllowed=self.config.widthStdAllowed) 

420 

421 if display and plotMagSize: 

422 fig = plot(mag, width, centers, clusterId, 

423 magType=self.config.sourceFluxField.split(".")[-1].title(), 

424 marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True) 

425 else: 

426 fig = None 

427 

428 clusterId = _improveCluster(width, centers, clusterId, 

429 nsigma=self.config.nSigmaClip, 

430 widthStdAllowed=self.config.widthStdAllowed) 

431 

432 if display and plotMagSize: 

433 plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False) 

434 

435 stellar = (clusterId == 0) 

436 # 

437 # We know enough to plot, if so requested 

438 # 

439 frame = 0 

440 

441 if fig: 

442 if display and displayExposure: 

443 disp = afwDisplay.Display(frame=frame) 

444 disp.mtv(exposure.getMaskedImage(), title="PSF candidates") 

445 

446 global eventHandler 

447 eventHandler = EventHandler(fig.get_axes()[0], mag, width, 

448 sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame]) 

449 

450 fig.show() 

451 

452 while True: 

453 try: 

454 reply = input("continue? [c h(elp) q(uit) p(db)] ").strip() 

455 except EOFError: 

456 reply = None 

457 if not reply: 

458 reply = "c" 

459 

460 if reply: 

461 if reply[0] == "h": 

462 print("""\ 

463 We cluster the points; red are the stellar candidates and the other colours are other clusters. 

464 Points labelled + are rejects from the cluster (only for cluster 0). 

465 

466 At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text 

467 

468 If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the 

469 image display. 

470 """) 

471 elif reply[0] == "p": 

472 import pdb 

473 pdb.set_trace() 

474 elif reply[0] == 'q': 

475 sys.exit(1) 

476 else: 

477 break 

478 

479 if display and displayExposure: 

480 mi = exposure.getMaskedImage() 

481 with disp.Buffering(): 

482 for i, source in enumerate(sourceCat): 

483 if good[i]: 

484 ctype = afwDisplay.GREEN # star candidate 

485 else: 

486 ctype = afwDisplay.RED # not star 

487 

488 disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) 

489 

490 # stellar only applies to good==True objects 

491 mask = good == True # noqa (numpy bool comparison): E712 

492 good[mask] = stellar 

493 

494 return Struct(selected=good)