Coverage for python/lsst/meas/algorithms/objectSizeStarSelector.py: 14%

253 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-23 03:15 -0800

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["ObjectSizeStarSelectorConfig", "ObjectSizeStarSelectorTask"] 

23 

24import sys 

25 

26import numpy 

27import warnings 

28from functools import reduce 

29 

30from lsst.utils.logging import getLogger 

31from lsst.pipe.base import Struct 

32import lsst.geom 

33from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS 

34import lsst.afw.geom as afwGeom 

35import lsst.pex.config as pexConfig 

36import lsst.afw.display as afwDisplay 

37from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry 

38 

39afwDisplay.setDefaultMaskTransparency(75) 

40 

41_LOG = getLogger(__name__) 

42 

43 

44class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass): 

45 doFluxLimit = pexConfig.Field( 

46 doc="Apply flux limit to Psf Candidate selection?", 

47 dtype=bool, 

48 default=True, 

49 ) 

50 fluxMin = pexConfig.Field( 50 ↛ exitline 50 didn't jump to the function exit

51 doc="specify the minimum psfFlux for good Psf Candidates", 

52 dtype=float, 

53 default=12500.0, 

54 check=lambda x: x >= 0.0, 

55 ) 

56 fluxMax = pexConfig.Field( 56 ↛ exitline 56 didn't jump to the function exit

57 doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)", 

58 dtype=float, 

59 default=0.0, 

60 check=lambda x: x >= 0.0, 

61 ) 

62 doSignalToNoiseLimit = pexConfig.Field( 

63 doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?", 

64 dtype=bool, 

65 default=False, 

66 ) 

67 # Note that the current default is conditioned on the detection thresholds 

68 # set in the characterizeImage setDefaults function for the measurePsf 

69 # stage. 

70 signalToNoiseMin = pexConfig.Field( 70 ↛ exitline 70 didn't jump to the function exit

71 doc="specify the minimum signal-to-noise for good Psf Candidates " 

72 "(value should take into consideration the detection thresholds " 

73 "set for the catalog of interest)", 

74 dtype=float, 

75 default=50.0, 

76 check=lambda x: x >= 0.0, 

77 ) 

78 signalToNoiseMax = pexConfig.Field( 78 ↛ exitline 78 didn't jump to the function exit

79 doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)", 

80 dtype=float, 

81 default=0.0, 

82 check=lambda x: x >= 0.0, 

83 ) 

84 widthMin = pexConfig.Field( 84 ↛ exitline 84 didn't jump to the function exit

85 doc="minimum width to include in histogram", 

86 dtype=float, 

87 default=0.0, 

88 check=lambda x: x >= 0.0, 

89 ) 

90 widthMax = pexConfig.Field( 90 ↛ exitline 90 didn't jump to the function exit

91 doc="maximum width to include in histogram", 

92 dtype=float, 

93 default=10.0, 

94 check=lambda x: x >= 0.0, 

95 ) 

96 sourceFluxField = pexConfig.Field( 

97 doc="Name of field in Source to use for flux measurement", 

98 dtype=str, 

99 default="base_GaussianFlux_instFlux", 

100 ) 

101 widthStdAllowed = pexConfig.Field( 101 ↛ exitline 101 didn't jump to the function exit

102 doc="Standard deviation of width allowed to be interpreted as good stars", 

103 dtype=float, 

104 default=0.15, 

105 check=lambda x: x >= 0.0, 

106 ) 

107 nSigmaClip = pexConfig.Field( 107 ↛ exitline 107 didn't jump to the function exit

108 doc="Keep objects within this many sigma of cluster 0's median", 

109 dtype=float, 

110 default=2.0, 

111 check=lambda x: x >= 0.0, 

112 ) 

113 badFlags = pexConfig.ListField( 

114 doc="List of flags which cause a source to be rejected as bad", 

115 dtype=str, 

116 default=[ 

117 "base_PixelFlags_flag_edge", 

118 "base_PixelFlags_flag_interpolatedCenter", 

119 "base_PixelFlags_flag_saturatedCenter", 

120 "base_PixelFlags_flag_crCenter", 

121 "base_PixelFlags_flag_bad", 

122 "base_PixelFlags_flag_interpolated", 

123 ], 

124 ) 

125 

126 def validate(self): 

127 BaseSourceSelectorTask.ConfigClass.validate(self) 

128 if self.widthMin > self.widthMax: 

129 msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})" 

130 raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg) 

131 

132 

133class EventHandler: 

134 """A class to handle key strokes with matplotlib displays. 

135 """ 

136 

137 def __init__(self, axes, xs, ys, x, y, frames=[0]): 

138 self.axes = axes 

139 self.xs = xs 

140 self.ys = ys 

141 self.x = x 

142 self.y = y 

143 self.frames = frames 

144 

145 self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self) 

146 

147 def __call__(self, ev): 

148 if ev.inaxes != self.axes: 

149 return 

150 

151 if ev.key and ev.key in ("p"): 

152 dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata) 

153 dist[numpy.where(numpy.isnan(dist))] = 1e30 

154 

155 which = numpy.where(dist == min(dist)) 

156 

157 x = self.x[which][0] 

158 y = self.y[which][0] 

159 for frame in self.frames: 

160 disp = afwDisplay.Display(frame=frame) 

161 disp.pan(x, y) 

162 disp.flush() 

163 else: 

164 pass 

165 

166 

167def _assignClusters(yvec, centers): 

168 """Return a vector of centerIds based on their distance to the centers. 

169 """ 

170 assert len(centers) > 0 

171 

172 minDist = numpy.nan*numpy.ones_like(yvec) 

173 clusterId = numpy.empty_like(yvec) 

174 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

175 dbl = _LOG.getChild("_assignClusters") 

176 dbl.setLevel(dbl.INFO) 

177 

178 # Make sure we are logging aall numpy warnings... 

179 oldSettings = numpy.seterr(all="warn") 

180 with warnings.catch_warnings(record=True) as w: 

181 warnings.simplefilter("always") 

182 for i, mean in enumerate(centers): 

183 dist = abs(yvec - mean) 

184 if i == 0: 

185 update = dist == dist # True for all points 

186 else: 

187 update = dist < minDist 

188 if w: # Only do if w is not empty i.e. contains a warning message 

189 dbl.trace(str(w[-1])) 

190 

191 minDist[update] = dist[update] 

192 clusterId[update] = i 

193 numpy.seterr(**oldSettings) 

194 

195 return clusterId 

196 

197 

198def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15): 

199 """A classic k-means algorithm, clustering yvec into nCluster clusters 

200 

201 Return the set of centres, and the cluster ID for each of the points 

202 

203 If useMedian is true, use the median of the cluster as its centre, rather than 

204 the traditional mean 

205 

206 Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means: 

207 "e.g. why not use the Forgy or random partition initialization methods" 

208 however, the approach adopted here seems to work well for the particular sorts of things 

209 we're clustering in this application 

210 """ 

211 

212 assert nCluster > 0 

213 

214 mean0 = sorted(yvec)[len(yvec)//10] # guess 

215 delta = mean0 * widthStdAllowed * 2.0 

216 centers = mean0 + delta * numpy.arange(nCluster) 

217 

218 func = numpy.median if useMedian else numpy.mean 

219 

220 clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to 

221 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

222 while True: 

223 oclusterId = clusterId 

224 clusterId = _assignClusters(yvec, centers) 

225 

226 if numpy.all(clusterId == oclusterId): 

227 break 

228 

229 for i in range(nCluster): 

230 # Only compute func if some points are available; otherwise, default to NaN. 

231 pointsInCluster = (clusterId == i) 

232 if numpy.any(pointsInCluster): 

233 centers[i] = func(yvec[pointsInCluster]) 

234 else: 

235 centers[i] = numpy.nan 

236 

237 return centers, clusterId 

238 

239 

240def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15): 

241 """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median. 

242 """ 

243 

244 nMember = sum(clusterId == clusterNum) 

245 if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement 

246 return clusterId 

247 for iter in range(nIteration): 

248 old_nMember = nMember 

249 

250 inCluster0 = clusterId == clusterNum 

251 yv = yvec[inCluster0] 

252 

253 centers[clusterNum] = numpy.median(yv) 

254 stdev = numpy.std(yv) 

255 

256 syv = sorted(yv) 

257 stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)]) 

258 median = syv[int(0.5*nMember)] 

259 

260 sd = stdev if stdev < stdev_iqr else stdev_iqr 

261 

262 if False: 

263 print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv))) 

264 newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd 

265 clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum 

266 clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1 

267 

268 nMember = sum(clusterId == clusterNum) 

269 # 'sd < widthStdAllowed * median' prevents too much rejections 

270 if nMember == old_nMember or sd < widthStdAllowed * median: 

271 break 

272 

273 return clusterId 

274 

275 

276def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', 

277 magType="model", clear=True): 

278 

279 log = _LOG.getChild("plot") 

280 try: 

281 import matplotlib.pyplot as plt 

282 except ImportError as e: 

283 log.warning("Unable to import matplotlib: %s", e) 

284 return 

285 

286 try: 

287 fig 

288 except NameError: 

289 fig = plt.figure() 

290 else: 

291 if clear: 

292 fig.clf() 

293 

294 axes = fig.add_axes((0.1, 0.1, 0.85, 0.80)) 

295 

296 xmin = sorted(mag)[int(0.05*len(mag))] 

297 xmax = sorted(mag)[int(0.95*len(mag))] 

298 

299 axes.set_xlim(-17.5, -13) 

300 axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin)) 

301 axes.set_ylim(0, 10) 

302 

303 colors = ["r", "g", "b", "c", "m", "k", ] 

304 for k, mean in enumerate(centers): 

305 if k == 0: 

306 axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype) 

307 

308 li = (clusterId == k) 

309 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

310 color=colors[k % len(colors)]) 

311 

312 li = (clusterId == -1) 

313 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

314 color='k') 

315 

316 if clear: 

317 axes.set_xlabel("Instrumental %s mag" % magType) 

318 axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$") 

319 

320 return fig 

321 

322 

323@pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry) 

324class ObjectSizeStarSelectorTask(BaseSourceSelectorTask): 

325 r"""A star selector that looks for a cluster of small objects in a size-magnitude plot. 

326 """ 

327 ConfigClass = ObjectSizeStarSelectorConfig 

328 usesMatches = False # selectStars does not use its matches argument 

329 

330 def selectSources(self, sourceCat, matches=None, exposure=None): 

331 """Return a selection of PSF candidates that represent likely stars. 

332 

333 A list of PSF candidates may be used by a PSF fitter to construct a PSF. 

334 

335 Parameters 

336 ---------- 

337 sourceCat : `lsst.afw.table.SourceCatalog` 

338 Catalog of sources to select from. 

339 This catalog must be contiguous in memory. 

340 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

341 Ignored in this SourceSelector. 

342 exposure : `lsst.afw.image.Exposure` or None 

343 The exposure the catalog was built from; used to get the detector 

344 to transform to TanPix, and for debug display. 

345 

346 Returns 

347 ------- 

348 struct : `lsst.pipe.base.Struct` 

349 The struct contains the following data: 

350 

351 ``selected`` 

352 Boolean array of sources that were selected, same length as 

353 sourceCat. (`numpy.ndarray` of `bool`) 

354 """ 

355 import lsstDebug 

356 display = lsstDebug.Info(__name__).display 

357 displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

358 plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation 

359 dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file? 

360 

361 detector = None 

362 pixToTanPix = None 

363 if exposure: 

364 detector = exposure.getDetector() 

365 if detector: 

366 pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS) 

367 # 

368 # Look at the distribution of stars in the magnitude-size plane 

369 # 

370 flux = sourceCat.get(self.config.sourceFluxField) 

371 fluxErr = sourceCat.get(self.config.sourceFluxField + "Err") 

372 

373 xx = numpy.empty(len(sourceCat)) 

374 xy = numpy.empty_like(xx) 

375 yy = numpy.empty_like(xx) 

376 for i, source in enumerate(sourceCat): 

377 Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy() 

378 if pixToTanPix: 

379 p = lsst.geom.Point2D(source.getX(), source.getY()) 

380 linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear() 

381 m = afwGeom.Quadrupole(Ixx, Iyy, Ixy) 

382 m.transform(linTransform) 

383 Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy() 

384 

385 xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy 

386 

387 width = numpy.sqrt(0.5*(xx + yy)) 

388 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

389 bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) 

390 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width))) 

391 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux))) 

392 if self.config.doFluxLimit: 

393 bad = numpy.logical_or(bad, flux < self.config.fluxMin) 

394 if self.config.fluxMax > 0: 

395 bad = numpy.logical_or(bad, flux > self.config.fluxMax) 

396 if self.config.doSignalToNoiseLimit: 

397 bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin) 

398 if self.config.signalToNoiseMax > 0: 

399 bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax) 

400 bad = numpy.logical_or(bad, width < self.config.widthMin) 

401 bad = numpy.logical_or(bad, width > self.config.widthMax) 

402 good = numpy.logical_not(bad) 

403 

404 if not numpy.any(good): 

405 raise RuntimeError("No objects passed our cuts for consideration as psf stars") 

406 

407 mag = -2.5*numpy.log10(flux[good]) 

408 width = width[good] 

409 # 

410 # Look for the maximum in the size histogram, then search upwards for the minimum that separates 

411 # the initial peak (of, we presume, stars) from the galaxies 

412 # 

413 if dumpData: 

414 import os 

415 import pickle as pickle 

416 _ii = 0 

417 while True: 

418 pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii)) 

419 if not os.path.exists(pickleFile): 

420 break 

421 _ii += 1 

422 

423 with open(pickleFile, "wb") as fd: 

424 pickle.dump(mag, fd, -1) 

425 pickle.dump(width, fd, -1) 

426 

427 centers, clusterId = _kcenters(width, nCluster=4, useMedian=True, 

428 widthStdAllowed=self.config.widthStdAllowed) 

429 

430 if display and plotMagSize: 

431 fig = plot(mag, width, centers, clusterId, 

432 magType=self.config.sourceFluxField.split(".")[-1].title(), 

433 marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True) 

434 else: 

435 fig = None 

436 

437 clusterId = _improveCluster(width, centers, clusterId, 

438 nsigma=self.config.nSigmaClip, 

439 widthStdAllowed=self.config.widthStdAllowed) 

440 

441 if display and plotMagSize: 

442 plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False) 

443 

444 stellar = (clusterId == 0) 

445 # 

446 # We know enough to plot, if so requested 

447 # 

448 frame = 0 

449 

450 if fig: 

451 if display and displayExposure: 

452 disp = afwDisplay.Display(frame=frame) 

453 disp.mtv(exposure.getMaskedImage(), title="PSF candidates") 

454 

455 global eventHandler 

456 eventHandler = EventHandler(fig.get_axes()[0], mag, width, 

457 sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame]) 

458 

459 fig.show() 

460 

461 while True: 

462 try: 

463 reply = input("continue? [c h(elp) q(uit) p(db)] ").strip() 

464 except EOFError: 

465 reply = None 

466 if not reply: 

467 reply = "c" 

468 

469 if reply: 

470 if reply[0] == "h": 

471 print("""\ 

472 We cluster the points; red are the stellar candidates and the other colours are other clusters. 

473 Points labelled + are rejects from the cluster (only for cluster 0). 

474 

475 At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text 

476 

477 If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the 

478 image display. 

479 """) 

480 elif reply[0] == "p": 

481 import pdb 

482 pdb.set_trace() 

483 elif reply[0] == 'q': 

484 sys.exit(1) 

485 else: 

486 break 

487 

488 if display and displayExposure: 

489 mi = exposure.getMaskedImage() 

490 with disp.Buffering(): 

491 for i, source in enumerate(sourceCat): 

492 if good[i]: 

493 ctype = afwDisplay.GREEN # star candidate 

494 else: 

495 ctype = afwDisplay.RED # not star 

496 

497 disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) 

498 

499 # stellar only applies to good==True objects 

500 mask = good == True # noqa (numpy bool comparison): E712 

501 good[mask] = stellar 

502 

503 return Struct(selected=good)