Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["ObjectSizeStarSelectorConfig", "ObjectSizeStarSelectorTask"] 

23 

24import sys 

25 

26import numpy 

27import warnings 

28from functools import reduce 

29 

30from lsst.log import Log 

31from lsst.pipe.base import Struct 

32import lsst.geom 

33from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS 

34import lsst.afw.geom as afwGeom 

35import lsst.pex.config as pexConfig 

36import lsst.afw.display as afwDisplay 

37from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry 

38 

39afwDisplay.setDefaultMaskTransparency(75) 

40 

41 

42class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass): 

43 doFluxLimit = pexConfig.Field( 

44 doc="Apply flux limit to Psf Candidate selection?", 

45 dtype=bool, 

46 default=True, 

47 ) 

48 fluxMin = pexConfig.Field( 48 ↛ exitline 48 didn't jump to the function exit

49 doc="specify the minimum psfFlux for good Psf Candidates", 

50 dtype=float, 

51 default=12500.0, 

52 check=lambda x: x >= 0.0, 

53 ) 

54 fluxMax = pexConfig.Field( 54 ↛ exitline 54 didn't jump to the function exit

55 doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)", 

56 dtype=float, 

57 default=0.0, 

58 check=lambda x: x >= 0.0, 

59 ) 

60 doSignalToNoiseLimit = pexConfig.Field( 

61 doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?", 

62 dtype=bool, 

63 default=False, 

64 ) 

65 signalToNoiseMin = pexConfig.Field( 65 ↛ exitline 65 didn't jump to the function exit

66 doc="specify the minimum signal-to-noise for good Psf Candidates", 

67 dtype=float, 

68 default=20.0, 

69 check=lambda x: x >= 0.0, 

70 ) 

71 signalToNoiseMax = pexConfig.Field( 71 ↛ exitline 71 didn't jump to the function exit

72 doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)", 

73 dtype=float, 

74 default=0.0, 

75 check=lambda x: x >= 0.0, 

76 ) 

77 widthMin = pexConfig.Field( 77 ↛ exitline 77 didn't jump to the function exit

78 doc="minimum width to include in histogram", 

79 dtype=float, 

80 default=0.0, 

81 check=lambda x: x >= 0.0, 

82 ) 

83 widthMax = pexConfig.Field( 83 ↛ exitline 83 didn't jump to the function exit

84 doc="maximum width to include in histogram", 

85 dtype=float, 

86 default=10.0, 

87 check=lambda x: x >= 0.0, 

88 ) 

89 sourceFluxField = pexConfig.Field( 

90 doc="Name of field in Source to use for flux measurement", 

91 dtype=str, 

92 default="base_GaussianFlux_instFlux", 

93 ) 

94 widthStdAllowed = pexConfig.Field( 94 ↛ exitline 94 didn't jump to the function exit

95 doc="Standard deviation of width allowed to be interpreted as good stars", 

96 dtype=float, 

97 default=0.15, 

98 check=lambda x: x >= 0.0, 

99 ) 

100 nSigmaClip = pexConfig.Field( 100 ↛ exitline 100 didn't jump to the function exit

101 doc="Keep objects within this many sigma of cluster 0's median", 

102 dtype=float, 

103 default=2.0, 

104 check=lambda x: x >= 0.0, 

105 ) 

106 badFlags = pexConfig.ListField( 

107 doc="List of flags which cause a source to be rejected as bad", 

108 dtype=str, 

109 default=[ 

110 "base_PixelFlags_flag_edge", 

111 "base_PixelFlags_flag_interpolatedCenter", 

112 "base_PixelFlags_flag_saturatedCenter", 

113 "base_PixelFlags_flag_crCenter", 

114 "base_PixelFlags_flag_bad", 

115 "base_PixelFlags_flag_interpolated", 

116 ], 

117 ) 

118 

119 def validate(self): 

120 BaseSourceSelectorTask.ConfigClass.validate(self) 

121 if self.widthMin > self.widthMax: 

122 msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})" 

123 raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg) 

124 

125 

126class EventHandler: 

127 """A class to handle key strokes with matplotlib displays. 

128 """ 

129 

130 def __init__(self, axes, xs, ys, x, y, frames=[0]): 

131 self.axes = axes 

132 self.xs = xs 

133 self.ys = ys 

134 self.x = x 

135 self.y = y 

136 self.frames = frames 

137 

138 self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self) 

139 

140 def __call__(self, ev): 

141 if ev.inaxes != self.axes: 

142 return 

143 

144 if ev.key and ev.key in ("p"): 

145 dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata) 

146 dist[numpy.where(numpy.isnan(dist))] = 1e30 

147 

148 which = numpy.where(dist == min(dist)) 

149 

150 x = self.x[which][0] 

151 y = self.y[which][0] 

152 for frame in self.frames: 

153 disp = afwDisplay.Display(frame=frame) 

154 disp.pan(x, y) 

155 disp.flush() 

156 else: 

157 pass 

158 

159 

160def _assignClusters(yvec, centers): 

161 """Return a vector of centerIds based on their distance to the centers. 

162 """ 

163 assert len(centers) > 0 

164 

165 minDist = numpy.nan*numpy.ones_like(yvec) 

166 clusterId = numpy.empty_like(yvec) 

167 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

168 dbl = Log.getLogger("objectSizeStarSelector._assignClusters") 

169 dbl.setLevel(dbl.INFO) 

170 

171 # Make sure we are logging aall numpy warnings... 

172 oldSettings = numpy.seterr(all="warn") 

173 with warnings.catch_warnings(record=True) as w: 

174 warnings.simplefilter("always") 

175 for i, mean in enumerate(centers): 

176 dist = abs(yvec - mean) 

177 if i == 0: 

178 update = dist == dist # True for all points 

179 else: 

180 update = dist < minDist 

181 if w: # Only do if w is not empty i.e. contains a warning message 

182 dbl.trace(str(w[-1])) 

183 

184 minDist[update] = dist[update] 

185 clusterId[update] = i 

186 numpy.seterr(**oldSettings) 

187 

188 return clusterId 

189 

190 

191def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15): 

192 """A classic k-means algorithm, clustering yvec into nCluster clusters 

193 

194 Return the set of centres, and the cluster ID for each of the points 

195 

196 If useMedian is true, use the median of the cluster as its centre, rather than 

197 the traditional mean 

198 

199 Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means: 

200 "e.g. why not use the Forgy or random partition initialization methods" 

201 however, the approach adopted here seems to work well for the particular sorts of things 

202 we're clustering in this application 

203 """ 

204 

205 assert nCluster > 0 

206 

207 mean0 = sorted(yvec)[len(yvec)//10] # guess 

208 delta = mean0 * widthStdAllowed * 2.0 

209 centers = mean0 + delta * numpy.arange(nCluster) 

210 

211 func = numpy.median if useMedian else numpy.mean 

212 

213 clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to 

214 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

215 while True: 

216 oclusterId = clusterId 

217 clusterId = _assignClusters(yvec, centers) 

218 

219 if numpy.all(clusterId == oclusterId): 

220 break 

221 

222 for i in range(nCluster): 

223 # Only compute func if some points are available; otherwise, default to NaN. 

224 pointsInCluster = (clusterId == i) 

225 if numpy.any(pointsInCluster): 

226 centers[i] = func(yvec[pointsInCluster]) 

227 else: 

228 centers[i] = numpy.nan 

229 

230 return centers, clusterId 

231 

232 

233def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15): 

234 """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median. 

235 """ 

236 

237 nMember = sum(clusterId == clusterNum) 

238 if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement 

239 return clusterId 

240 for iter in range(nIteration): 

241 old_nMember = nMember 

242 

243 inCluster0 = clusterId == clusterNum 

244 yv = yvec[inCluster0] 

245 

246 centers[clusterNum] = numpy.median(yv) 

247 stdev = numpy.std(yv) 

248 

249 syv = sorted(yv) 

250 stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)]) 

251 median = syv[int(0.5*nMember)] 

252 

253 sd = stdev if stdev < stdev_iqr else stdev_iqr 

254 

255 if False: 

256 print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv))) 

257 newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd 

258 clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum 

259 clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1 

260 

261 nMember = sum(clusterId == clusterNum) 

262 # 'sd < widthStdAllowed * median' prevents too much rejections 

263 if nMember == old_nMember or sd < widthStdAllowed * median: 

264 break 

265 

266 return clusterId 

267 

268 

269def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', 

270 magType="model", clear=True): 

271 

272 log = Log.getLogger("objectSizeStarSelector.plot") 

273 try: 

274 import matplotlib.pyplot as plt 

275 except ImportError as e: 

276 log.warning("Unable to import matplotlib: %s", e) 

277 return 

278 

279 try: 

280 fig 

281 except NameError: 

282 fig = plt.figure() 

283 else: 

284 if clear: 

285 fig.clf() 

286 

287 axes = fig.add_axes((0.1, 0.1, 0.85, 0.80)) 

288 

289 xmin = sorted(mag)[int(0.05*len(mag))] 

290 xmax = sorted(mag)[int(0.95*len(mag))] 

291 

292 axes.set_xlim(-17.5, -13) 

293 axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin)) 

294 axes.set_ylim(0, 10) 

295 

296 colors = ["r", "g", "b", "c", "m", "k", ] 

297 for k, mean in enumerate(centers): 

298 if k == 0: 

299 axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype) 

300 

301 li = (clusterId == k) 

302 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

303 color=colors[k % len(colors)]) 

304 

305 li = (clusterId == -1) 

306 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

307 color='k') 

308 

309 if clear: 

310 axes.set_xlabel("Instrumental %s mag" % magType) 

311 axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$") 

312 

313 return fig 

314 

315 

316@pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry) 

317class ObjectSizeStarSelectorTask(BaseSourceSelectorTask): 

318 r"""A star selector that looks for a cluster of small objects in a size-magnitude plot. 

319 """ 

320 ConfigClass = ObjectSizeStarSelectorConfig 

321 usesMatches = False # selectStars does not use its matches argument 

322 

323 def selectSources(self, sourceCat, matches=None, exposure=None): 

324 """Return a selection of PSF candidates that represent likely stars. 

325 

326 A list of PSF candidates may be used by a PSF fitter to construct a PSF. 

327 

328 Parameters: 

329 ----------- 

330 sourceCat : `lsst.afw.table.SourceCatalog` 

331 Catalog of sources to select from. 

332 This catalog must be contiguous in memory. 

333 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

334 Ignored in this SourceSelector. 

335 exposure : `lsst.afw.image.Exposure` or None 

336 The exposure the catalog was built from; used to get the detector 

337 to transform to TanPix, and for debug display. 

338 

339 Return 

340 ------ 

341 struct : `lsst.pipe.base.Struct` 

342 The struct contains the following data: 

343 

344 - selected : `array` of `bool`` 

345 Boolean array of sources that were selected, same length as 

346 sourceCat. 

347 """ 

348 import lsstDebug 

349 display = lsstDebug.Info(__name__).display 

350 displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

351 plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation 

352 dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file? 

353 

354 detector = None 

355 pixToTanPix = None 

356 if exposure: 

357 detector = exposure.getDetector() 

358 if detector: 

359 pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS) 

360 # 

361 # Look at the distribution of stars in the magnitude-size plane 

362 # 

363 flux = sourceCat.get(self.config.sourceFluxField) 

364 fluxErr = sourceCat.get(self.config.sourceFluxField + "Err") 

365 

366 xx = numpy.empty(len(sourceCat)) 

367 xy = numpy.empty_like(xx) 

368 yy = numpy.empty_like(xx) 

369 for i, source in enumerate(sourceCat): 

370 Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy() 

371 if pixToTanPix: 

372 p = lsst.geom.Point2D(source.getX(), source.getY()) 

373 linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear() 

374 m = afwGeom.Quadrupole(Ixx, Iyy, Ixy) 

375 m.transform(linTransform) 

376 Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy() 

377 

378 xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy 

379 

380 width = numpy.sqrt(0.5*(xx + yy)) 

381 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

382 bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) 

383 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width))) 

384 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux))) 

385 if self.config.doFluxLimit: 

386 bad = numpy.logical_or(bad, flux < self.config.fluxMin) 

387 if self.config.fluxMax > 0: 

388 bad = numpy.logical_or(bad, flux > self.config.fluxMax) 

389 if self.config.doSignalToNoiseLimit: 

390 bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin) 

391 if self.config.signalToNoiseMax > 0: 

392 bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax) 

393 bad = numpy.logical_or(bad, width < self.config.widthMin) 

394 bad = numpy.logical_or(bad, width > self.config.widthMax) 

395 good = numpy.logical_not(bad) 

396 

397 if not numpy.any(good): 

398 raise RuntimeError("No objects passed our cuts for consideration as psf stars") 

399 

400 mag = -2.5*numpy.log10(flux[good]) 

401 width = width[good] 

402 # 

403 # Look for the maximum in the size histogram, then search upwards for the minimum that separates 

404 # the initial peak (of, we presume, stars) from the galaxies 

405 # 

406 if dumpData: 

407 import os 

408 import pickle as pickle 

409 _ii = 0 

410 while True: 

411 pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii)) 

412 if not os.path.exists(pickleFile): 

413 break 

414 _ii += 1 

415 

416 with open(pickleFile, "wb") as fd: 

417 pickle.dump(mag, fd, -1) 

418 pickle.dump(width, fd, -1) 

419 

420 centers, clusterId = _kcenters(width, nCluster=4, useMedian=True, 

421 widthStdAllowed=self.config.widthStdAllowed) 

422 

423 if display and plotMagSize: 

424 fig = plot(mag, width, centers, clusterId, 

425 magType=self.config.sourceFluxField.split(".")[-1].title(), 

426 marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True) 

427 else: 

428 fig = None 

429 

430 clusterId = _improveCluster(width, centers, clusterId, 

431 nsigma=self.config.nSigmaClip, 

432 widthStdAllowed=self.config.widthStdAllowed) 

433 

434 if display and plotMagSize: 

435 plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False) 

436 

437 stellar = (clusterId == 0) 

438 # 

439 # We know enough to plot, if so requested 

440 # 

441 frame = 0 

442 

443 if fig: 

444 if display and displayExposure: 

445 disp = afwDisplay.Display(frame=frame) 

446 disp.mtv(exposure.getMaskedImage(), title="PSF candidates") 

447 

448 global eventHandler 

449 eventHandler = EventHandler(fig.get_axes()[0], mag, width, 

450 sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame]) 

451 

452 fig.show() 

453 

454 while True: 

455 try: 

456 reply = input("continue? [c h(elp) q(uit) p(db)] ").strip() 

457 except EOFError: 

458 reply = None 

459 if not reply: 

460 reply = "c" 

461 

462 if reply: 

463 if reply[0] == "h": 

464 print("""\ 

465 We cluster the points; red are the stellar candidates and the other colours are other clusters. 

466 Points labelled + are rejects from the cluster (only for cluster 0). 

467 

468 At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text 

469 

470 If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the 

471 image display. 

472 """) 

473 elif reply[0] == "p": 

474 import pdb 

475 pdb.set_trace() 

476 elif reply[0] == 'q': 

477 sys.exit(1) 

478 else: 

479 break 

480 

481 if display and displayExposure: 

482 mi = exposure.getMaskedImage() 

483 with disp.Buffering(): 

484 for i, source in enumerate(sourceCat): 

485 if good[i]: 

486 ctype = afwDisplay.GREEN # star candidate 

487 else: 

488 ctype = afwDisplay.RED # not star 

489 

490 disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) 

491 

492 # stellar only applies to good==True objects 

493 mask = good == True # noqa (numpy bool comparison): E712 

494 good[mask] = stellar 

495 

496 return Struct(selected=good)