Coverage for python/lsst/meas/algorithms/objectSizeStarSelector.py: 14%

248 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:11 -0800

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["ObjectSizeStarSelectorConfig", "ObjectSizeStarSelectorTask"] 

23 

24import sys 

25 

26import numpy 

27import warnings 

28from functools import reduce 

29 

30from lsst.log import Log 

31from lsst.pipe.base import Struct 

32import lsst.geom 

33from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS 

34import lsst.afw.geom as afwGeom 

35import lsst.pex.config as pexConfig 

36import lsst.afw.display as afwDisplay 

37from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry 

38 

39afwDisplay.setDefaultMaskTransparency(75) 

40 

41 

42class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass): 

43 doFluxLimit = pexConfig.Field( 

44 doc="Apply flux limit to Psf Candidate selection?", 

45 dtype=bool, 

46 default=True, 

47 ) 

48 fluxMin = pexConfig.Field( 48 ↛ exitline 48 didn't jump to the function exit

49 doc="specify the minimum psfFlux for good Psf Candidates", 

50 dtype=float, 

51 default=12500.0, 

52 check=lambda x: x >= 0.0, 

53 ) 

54 fluxMax = pexConfig.Field( 54 ↛ exitline 54 didn't jump to the function exit

55 doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)", 

56 dtype=float, 

57 default=0.0, 

58 check=lambda x: x >= 0.0, 

59 ) 

60 doSignalToNoiseLimit = pexConfig.Field( 

61 doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?", 

62 dtype=bool, 

63 default=False, 

64 ) 

65 # Note that the current default is conditioned on the detection thresholds 

66 # set in the characterizeImage setDefaults function for the measurePsf 

67 # stage. 

68 signalToNoiseMin = pexConfig.Field( 68 ↛ exitline 68 didn't jump to the function exit

69 doc="specify the minimum signal-to-noise for good Psf Candidates " 

70 "(value should take into consideration the detection thresholds " 

71 "set for the catalog of interest)", 

72 dtype=float, 

73 default=50.0, 

74 check=lambda x: x >= 0.0, 

75 ) 

76 signalToNoiseMax = pexConfig.Field( 76 ↛ exitline 76 didn't jump to the function exit

77 doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)", 

78 dtype=float, 

79 default=0.0, 

80 check=lambda x: x >= 0.0, 

81 ) 

82 widthMin = pexConfig.Field( 82 ↛ exitline 82 didn't jump to the function exit

83 doc="minimum width to include in histogram", 

84 dtype=float, 

85 default=0.0, 

86 check=lambda x: x >= 0.0, 

87 ) 

88 widthMax = pexConfig.Field( 88 ↛ exitline 88 didn't jump to the function exit

89 doc="maximum width to include in histogram", 

90 dtype=float, 

91 default=10.0, 

92 check=lambda x: x >= 0.0, 

93 ) 

94 sourceFluxField = pexConfig.Field( 

95 doc="Name of field in Source to use for flux measurement", 

96 dtype=str, 

97 default="base_GaussianFlux_instFlux", 

98 ) 

99 widthStdAllowed = pexConfig.Field( 99 ↛ exitline 99 didn't jump to the function exit

100 doc="Standard deviation of width allowed to be interpreted as good stars", 

101 dtype=float, 

102 default=0.15, 

103 check=lambda x: x >= 0.0, 

104 ) 

105 nSigmaClip = pexConfig.Field( 105 ↛ exitline 105 didn't jump to the function exit

106 doc="Keep objects within this many sigma of cluster 0's median", 

107 dtype=float, 

108 default=2.0, 

109 check=lambda x: x >= 0.0, 

110 ) 

111 badFlags = pexConfig.ListField( 

112 doc="List of flags which cause a source to be rejected as bad", 

113 dtype=str, 

114 default=[ 

115 "base_PixelFlags_flag_edge", 

116 "base_PixelFlags_flag_interpolatedCenter", 

117 "base_PixelFlags_flag_saturatedCenter", 

118 "base_PixelFlags_flag_crCenter", 

119 "base_PixelFlags_flag_bad", 

120 "base_PixelFlags_flag_interpolated", 

121 ], 

122 ) 

123 

124 def validate(self): 

125 BaseSourceSelectorTask.ConfigClass.validate(self) 

126 if self.widthMin > self.widthMax: 

127 msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})" 

128 raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg) 

129 

130 

131class EventHandler: 

132 """A class to handle key strokes with matplotlib displays. 

133 """ 

134 

135 def __init__(self, axes, xs, ys, x, y, frames=[0]): 

136 self.axes = axes 

137 self.xs = xs 

138 self.ys = ys 

139 self.x = x 

140 self.y = y 

141 self.frames = frames 

142 

143 self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self) 

144 

145 def __call__(self, ev): 

146 if ev.inaxes != self.axes: 

147 return 

148 

149 if ev.key and ev.key in ("p"): 

150 dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata) 

151 dist[numpy.where(numpy.isnan(dist))] = 1e30 

152 

153 which = numpy.where(dist == min(dist)) 

154 

155 x = self.x[which][0] 

156 y = self.y[which][0] 

157 for frame in self.frames: 

158 disp = afwDisplay.Display(frame=frame) 

159 disp.pan(x, y) 

160 disp.flush() 

161 else: 

162 pass 

163 

164 

165def _assignClusters(yvec, centers): 

166 """Return a vector of centerIds based on their distance to the centers. 

167 """ 

168 assert len(centers) > 0 

169 

170 minDist = numpy.nan*numpy.ones_like(yvec) 

171 clusterId = numpy.empty_like(yvec) 

172 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

173 dbl = Log.getLogger("objectSizeStarSelector._assignClusters") 

174 dbl.setLevel(dbl.INFO) 

175 

176 # Make sure we are logging aall numpy warnings... 

177 oldSettings = numpy.seterr(all="warn") 

178 with warnings.catch_warnings(record=True) as w: 

179 warnings.simplefilter("always") 

180 for i, mean in enumerate(centers): 

181 dist = abs(yvec - mean) 

182 if i == 0: 

183 update = dist == dist # True for all points 

184 else: 

185 update = dist < minDist 

186 if w: # Only do if w is not empty i.e. contains a warning message 

187 dbl.trace(str(w[-1])) 

188 

189 minDist[update] = dist[update] 

190 clusterId[update] = i 

191 numpy.seterr(**oldSettings) 

192 

193 return clusterId 

194 

195 

196def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15): 

197 """A classic k-means algorithm, clustering yvec into nCluster clusters 

198 

199 Return the set of centres, and the cluster ID for each of the points 

200 

201 If useMedian is true, use the median of the cluster as its centre, rather than 

202 the traditional mean 

203 

204 Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means: 

205 "e.g. why not use the Forgy or random partition initialization methods" 

206 however, the approach adopted here seems to work well for the particular sorts of things 

207 we're clustering in this application 

208 """ 

209 

210 assert nCluster > 0 

211 

212 mean0 = sorted(yvec)[len(yvec)//10] # guess 

213 delta = mean0 * widthStdAllowed * 2.0 

214 centers = mean0 + delta * numpy.arange(nCluster) 

215 

216 func = numpy.median if useMedian else numpy.mean 

217 

218 clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to 

219 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

220 while True: 

221 oclusterId = clusterId 

222 clusterId = _assignClusters(yvec, centers) 

223 

224 if numpy.all(clusterId == oclusterId): 

225 break 

226 

227 for i in range(nCluster): 

228 # Only compute func if some points are available; otherwise, default to NaN. 

229 pointsInCluster = (clusterId == i) 

230 if numpy.any(pointsInCluster): 

231 centers[i] = func(yvec[pointsInCluster]) 

232 else: 

233 centers[i] = numpy.nan 

234 

235 return centers, clusterId 

236 

237 

238def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15): 

239 """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median. 

240 """ 

241 

242 nMember = sum(clusterId == clusterNum) 

243 if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement 

244 return clusterId 

245 for iter in range(nIteration): 

246 old_nMember = nMember 

247 

248 inCluster0 = clusterId == clusterNum 

249 yv = yvec[inCluster0] 

250 

251 centers[clusterNum] = numpy.median(yv) 

252 stdev = numpy.std(yv) 

253 

254 syv = sorted(yv) 

255 stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)]) 

256 median = syv[int(0.5*nMember)] 

257 

258 sd = stdev if stdev < stdev_iqr else stdev_iqr 

259 

260 if False: 

261 print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv))) 

262 newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd 

263 clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum 

264 clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1 

265 

266 nMember = sum(clusterId == clusterNum) 

267 # 'sd < widthStdAllowed * median' prevents too much rejections 

268 if nMember == old_nMember or sd < widthStdAllowed * median: 

269 break 

270 

271 return clusterId 

272 

273 

274def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', 

275 magType="model", clear=True): 

276 

277 log = Log.getLogger("objectSizeStarSelector.plot") 

278 try: 

279 import matplotlib.pyplot as plt 

280 except ImportError as e: 

281 log.warning("Unable to import matplotlib: %s", e) 

282 return 

283 

284 try: 

285 fig 

286 except NameError: 

287 fig = plt.figure() 

288 else: 

289 if clear: 

290 fig.clf() 

291 

292 axes = fig.add_axes((0.1, 0.1, 0.85, 0.80)) 

293 

294 xmin = sorted(mag)[int(0.05*len(mag))] 

295 xmax = sorted(mag)[int(0.95*len(mag))] 

296 

297 axes.set_xlim(-17.5, -13) 

298 axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin)) 

299 axes.set_ylim(0, 10) 

300 

301 colors = ["r", "g", "b", "c", "m", "k", ] 

302 for k, mean in enumerate(centers): 

303 if k == 0: 

304 axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype) 

305 

306 li = (clusterId == k) 

307 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

308 color=colors[k % len(colors)]) 

309 

310 li = (clusterId == -1) 

311 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

312 color='k') 

313 

314 if clear: 

315 axes.set_xlabel("Instrumental %s mag" % magType) 

316 axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$") 

317 

318 return fig 

319 

320 

321@pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry) 

322class ObjectSizeStarSelectorTask(BaseSourceSelectorTask): 

323 r"""A star selector that looks for a cluster of small objects in a size-magnitude plot. 

324 """ 

325 ConfigClass = ObjectSizeStarSelectorConfig 

326 usesMatches = False # selectStars does not use its matches argument 

327 

328 def selectSources(self, sourceCat, matches=None, exposure=None): 

329 """Return a selection of PSF candidates that represent likely stars. 

330 

331 A list of PSF candidates may be used by a PSF fitter to construct a PSF. 

332 

333 Parameters: 

334 ----------- 

335 sourceCat : `lsst.afw.table.SourceCatalog` 

336 Catalog of sources to select from. 

337 This catalog must be contiguous in memory. 

338 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

339 Ignored in this SourceSelector. 

340 exposure : `lsst.afw.image.Exposure` or None 

341 The exposure the catalog was built from; used to get the detector 

342 to transform to TanPix, and for debug display. 

343 

344 Return 

345 ------ 

346 struct : `lsst.pipe.base.Struct` 

347 The struct contains the following data: 

348 

349 - selected : `array` of `bool`` 

350 Boolean array of sources that were selected, same length as 

351 sourceCat. 

352 """ 

353 import lsstDebug 

354 display = lsstDebug.Info(__name__).display 

355 displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

356 plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation 

357 dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file? 

358 

359 detector = None 

360 pixToTanPix = None 

361 if exposure: 

362 detector = exposure.getDetector() 

363 if detector: 

364 pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS) 

365 # 

366 # Look at the distribution of stars in the magnitude-size plane 

367 # 

368 flux = sourceCat.get(self.config.sourceFluxField) 

369 fluxErr = sourceCat.get(self.config.sourceFluxField + "Err") 

370 

371 xx = numpy.empty(len(sourceCat)) 

372 xy = numpy.empty_like(xx) 

373 yy = numpy.empty_like(xx) 

374 for i, source in enumerate(sourceCat): 

375 Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy() 

376 if pixToTanPix: 

377 p = lsst.geom.Point2D(source.getX(), source.getY()) 

378 linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear() 

379 m = afwGeom.Quadrupole(Ixx, Iyy, Ixy) 

380 m.transform(linTransform) 

381 Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy() 

382 

383 xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy 

384 

385 width = numpy.sqrt(0.5*(xx + yy)) 

386 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

387 bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) 

388 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width))) 

389 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux))) 

390 if self.config.doFluxLimit: 

391 bad = numpy.logical_or(bad, flux < self.config.fluxMin) 

392 if self.config.fluxMax > 0: 

393 bad = numpy.logical_or(bad, flux > self.config.fluxMax) 

394 if self.config.doSignalToNoiseLimit: 

395 bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin) 

396 if self.config.signalToNoiseMax > 0: 

397 bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax) 

398 bad = numpy.logical_or(bad, width < self.config.widthMin) 

399 bad = numpy.logical_or(bad, width > self.config.widthMax) 

400 good = numpy.logical_not(bad) 

401 

402 if not numpy.any(good): 

403 raise RuntimeError("No objects passed our cuts for consideration as psf stars") 

404 

405 mag = -2.5*numpy.log10(flux[good]) 

406 width = width[good] 

407 # 

408 # Look for the maximum in the size histogram, then search upwards for the minimum that separates 

409 # the initial peak (of, we presume, stars) from the galaxies 

410 # 

411 if dumpData: 

412 import os 

413 import pickle as pickle 

414 _ii = 0 

415 while True: 

416 pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii)) 

417 if not os.path.exists(pickleFile): 

418 break 

419 _ii += 1 

420 

421 with open(pickleFile, "wb") as fd: 

422 pickle.dump(mag, fd, -1) 

423 pickle.dump(width, fd, -1) 

424 

425 centers, clusterId = _kcenters(width, nCluster=4, useMedian=True, 

426 widthStdAllowed=self.config.widthStdAllowed) 

427 

428 if display and plotMagSize: 

429 fig = plot(mag, width, centers, clusterId, 

430 magType=self.config.sourceFluxField.split(".")[-1].title(), 

431 marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True) 

432 else: 

433 fig = None 

434 

435 clusterId = _improveCluster(width, centers, clusterId, 

436 nsigma=self.config.nSigmaClip, 

437 widthStdAllowed=self.config.widthStdAllowed) 

438 

439 if display and plotMagSize: 

440 plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False) 

441 

442 stellar = (clusterId == 0) 

443 # 

444 # We know enough to plot, if so requested 

445 # 

446 frame = 0 

447 

448 if fig: 

449 if display and displayExposure: 

450 disp = afwDisplay.Display(frame=frame) 

451 disp.mtv(exposure.getMaskedImage(), title="PSF candidates") 

452 

453 global eventHandler 

454 eventHandler = EventHandler(fig.get_axes()[0], mag, width, 

455 sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame]) 

456 

457 fig.show() 

458 

459 while True: 

460 try: 

461 reply = input("continue? [c h(elp) q(uit) p(db)] ").strip() 

462 except EOFError: 

463 reply = None 

464 if not reply: 

465 reply = "c" 

466 

467 if reply: 

468 if reply[0] == "h": 

469 print("""\ 

470 We cluster the points; red are the stellar candidates and the other colours are other clusters. 

471 Points labelled + are rejects from the cluster (only for cluster 0). 

472 

473 At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text 

474 

475 If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the 

476 image display. 

477 """) 

478 elif reply[0] == "p": 

479 import pdb 

480 pdb.set_trace() 

481 elif reply[0] == 'q': 

482 sys.exit(1) 

483 else: 

484 break 

485 

486 if display and displayExposure: 

487 mi = exposure.getMaskedImage() 

488 with disp.Buffering(): 

489 for i, source in enumerate(sourceCat): 

490 if good[i]: 

491 ctype = afwDisplay.GREEN # star candidate 

492 else: 

493 ctype = afwDisplay.RED # not star 

494 

495 disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) 

496 

497 # stellar only applies to good==True objects 

498 mask = good == True # noqa (numpy bool comparison): E712 

499 good[mask] = stellar 

500 

501 return Struct(selected=good)