Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import sys 

23 

24import numpy 

25import warnings 

26from functools import reduce 

27 

28from lsst.log import Log 

29from lsst.pipe.base import Struct 

30import lsst.geom 

31from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS 

32import lsst.afw.geom as afwGeom 

33import lsst.pex.config as pexConfig 

34import lsst.afw.display as afwDisplay 

35from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry 

36 

37afwDisplay.setDefaultMaskTransparency(75) 

38 

39 

40class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass): 

41 doFluxLimit = pexConfig.Field( 

42 doc="Apply flux limit to Psf Candidate selection?", 

43 dtype=bool, 

44 default=True, 

45 ) 

46 fluxMin = pexConfig.Field( 

47 doc="specify the minimum psfFlux for good Psf Candidates", 

48 dtype=float, 

49 default=12500.0, 

50 check=lambda x: x >= 0.0, 

51 ) 

52 fluxMax = pexConfig.Field( 

53 doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)", 

54 dtype=float, 

55 default=0.0, 

56 check=lambda x: x >= 0.0, 

57 ) 

58 doSignalToNoiseLimit = pexConfig.Field( 

59 doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?", 

60 dtype=bool, 

61 default=False, 

62 ) 

63 signalToNoiseMin = pexConfig.Field( 

64 doc="specify the minimum signal-to-noise for good Psf Candidates", 

65 dtype=float, 

66 default=20.0, 

67 check=lambda x: x >= 0.0, 

68 ) 

69 signalToNoiseMax = pexConfig.Field( 

70 doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)", 

71 dtype=float, 

72 default=0.0, 

73 check=lambda x: x >= 0.0, 

74 ) 

75 widthMin = pexConfig.Field( 

76 doc="minimum width to include in histogram", 

77 dtype=float, 

78 default=0.0, 

79 check=lambda x: x >= 0.0, 

80 ) 

81 widthMax = pexConfig.Field( 

82 doc="maximum width to include in histogram", 

83 dtype=float, 

84 default=10.0, 

85 check=lambda x: x >= 0.0, 

86 ) 

87 sourceFluxField = pexConfig.Field( 

88 doc="Name of field in Source to use for flux measurement", 

89 dtype=str, 

90 default="base_GaussianFlux_instFlux", 

91 ) 

92 widthStdAllowed = pexConfig.Field( 

93 doc="Standard deviation of width allowed to be interpreted as good stars", 

94 dtype=float, 

95 default=0.15, 

96 check=lambda x: x >= 0.0, 

97 ) 

98 nSigmaClip = pexConfig.Field( 

99 doc="Keep objects within this many sigma of cluster 0's median", 

100 dtype=float, 

101 default=2.0, 

102 check=lambda x: x >= 0.0, 

103 ) 

104 badFlags = pexConfig.ListField( 

105 doc="List of flags which cause a source to be rejected as bad", 

106 dtype=str, 

107 default=[ 

108 "base_PixelFlags_flag_edge", 

109 "base_PixelFlags_flag_interpolatedCenter", 

110 "base_PixelFlags_flag_saturatedCenter", 

111 "base_PixelFlags_flag_crCenter", 

112 "base_PixelFlags_flag_bad", 

113 "base_PixelFlags_flag_interpolated", 

114 ], 

115 ) 

116 

117 def validate(self): 

118 BaseSourceSelectorTask.ConfigClass.validate(self) 

119 if self.widthMin > self.widthMax: 

120 msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})" 

121 raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg) 

122 

123 

124class EventHandler: 

125 """A class to handle key strokes with matplotlib displays""" 

126 

127 def __init__(self, axes, xs, ys, x, y, frames=[0]): 

128 self.axes = axes 

129 self.xs = xs 

130 self.ys = ys 

131 self.x = x 

132 self.y = y 

133 self.frames = frames 

134 

135 self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self) 

136 

137 def __call__(self, ev): 

138 if ev.inaxes != self.axes: 

139 return 

140 

141 if ev.key and ev.key in ("p"): 

142 dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata) 

143 dist[numpy.where(numpy.isnan(dist))] = 1e30 

144 

145 which = numpy.where(dist == min(dist)) 

146 

147 x = self.x[which][0] 

148 y = self.y[which][0] 

149 for frame in self.frames: 

150 disp = afwDisplay.Display(frame=frame) 

151 disp.pan(x, y) 

152 disp.flush() 

153 else: 

154 pass 

155 

156 

157def _assignClusters(yvec, centers): 

158 """Return a vector of centerIds based on their distance to the centers""" 

159 assert len(centers) > 0 

160 

161 minDist = numpy.nan*numpy.ones_like(yvec) 

162 clusterId = numpy.empty_like(yvec) 

163 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

164 dbl = Log.getLogger("objectSizeStarSelector._assignClusters") 

165 dbl.setLevel(dbl.INFO) 

166 

167 # Make sure we are logging aall numpy warnings... 

168 oldSettings = numpy.seterr(all="warn") 

169 with warnings.catch_warnings(record=True) as w: 

170 warnings.simplefilter("always") 

171 for i, mean in enumerate(centers): 

172 dist = abs(yvec - mean) 

173 if i == 0: 

174 update = dist == dist # True for all points 

175 else: 

176 update = dist < minDist 

177 if w: # Only do if w is not empty i.e. contains a warning message 

178 dbl.trace(str(w[-1])) 

179 

180 minDist[update] = dist[update] 

181 clusterId[update] = i 

182 numpy.seterr(**oldSettings) 

183 

184 return clusterId 

185 

186 

187def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15): 

188 """A classic k-means algorithm, clustering yvec into nCluster clusters 

189 

190 Return the set of centres, and the cluster ID for each of the points 

191 

192 If useMedian is true, use the median of the cluster as its centre, rather than 

193 the traditional mean 

194 

195 Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means: 

196 "e.g. why not use the Forgy or random partition initialization methods" 

197 however, the approach adopted here seems to work well for the particular sorts of things 

198 we're clustering in this application 

199 """ 

200 

201 assert nCluster > 0 

202 

203 mean0 = sorted(yvec)[len(yvec)//10] # guess 

204 delta = mean0 * widthStdAllowed * 2.0 

205 centers = mean0 + delta * numpy.arange(nCluster) 

206 

207 func = numpy.median if useMedian else numpy.mean 

208 

209 clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to 

210 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5 

211 while True: 

212 oclusterId = clusterId 

213 clusterId = _assignClusters(yvec, centers) 

214 

215 if numpy.all(clusterId == oclusterId): 

216 break 

217 

218 for i in range(nCluster): 

219 # Only compute func if some points are available; otherwise, default to NaN. 

220 pointsInCluster = (clusterId == i) 

221 if numpy.any(pointsInCluster): 

222 centers[i] = func(yvec[pointsInCluster]) 

223 else: 

224 centers[i] = numpy.nan 

225 

226 return centers, clusterId 

227 

228 

229def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15): 

230 """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median""" 

231 

232 nMember = sum(clusterId == clusterNum) 

233 if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement 

234 return clusterId 

235 for iter in range(nIteration): 235 ↛ 261line 235 didn't jump to line 261, because the loop on line 235 didn't complete

236 old_nMember = nMember 

237 

238 inCluster0 = clusterId == clusterNum 

239 yv = yvec[inCluster0] 

240 

241 centers[clusterNum] = numpy.median(yv) 

242 stdev = numpy.std(yv) 

243 

244 syv = sorted(yv) 

245 stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)]) 

246 median = syv[int(0.5*nMember)] 

247 

248 sd = stdev if stdev < stdev_iqr else stdev_iqr 

249 

250 if False: 

251 print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv))) 

252 newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd 

253 clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum 

254 clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1 

255 

256 nMember = sum(clusterId == clusterNum) 

257 # 'sd < widthStdAllowed * median' prevents too much rejections 

258 if nMember == old_nMember or sd < widthStdAllowed * median: 258 ↛ 235line 258 didn't jump to line 235, because the condition on line 258 was never false

259 break 

260 

261 return clusterId 

262 

263 

264def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', 

265 magType="model", clear=True): 

266 

267 log = Log.getLogger("objectSizeStarSelector.plot") 

268 try: 

269 import matplotlib.pyplot as plt 

270 except ImportError as e: 

271 log.warn("Unable to import matplotlib: %s", e) 

272 return 

273 

274 try: 

275 fig 

276 except NameError: 

277 fig = plt.figure() 

278 else: 

279 if clear: 

280 fig.clf() 

281 

282 axes = fig.add_axes((0.1, 0.1, 0.85, 0.80)) 

283 

284 xmin = sorted(mag)[int(0.05*len(mag))] 

285 xmax = sorted(mag)[int(0.95*len(mag))] 

286 

287 axes.set_xlim(-17.5, -13) 

288 axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin)) 

289 axes.set_ylim(0, 10) 

290 

291 colors = ["r", "g", "b", "c", "m", "k", ] 

292 for k, mean in enumerate(centers): 

293 if k == 0: 

294 axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype) 

295 

296 li = (clusterId == k) 

297 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

298 color=colors[k % len(colors)]) 

299 

300 li = (clusterId == -1) 

301 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth, 

302 color='k') 

303 

304 if clear: 

305 axes.set_xlabel("Instrumental %s mag" % magType) 

306 axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$") 

307 

308 return fig 

309 

310## @addtogroup LSST_task_documentation 

311## @{ 

312## @page ObjectSizeStarSelectorTask 

313## @ref ObjectSizeStarSelectorTask_ "ObjectSizeStarSelectorTask" 

314## @copybrief ObjectSizeStarSelectorTask 

315## @} 

316 

317 

318@pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry) 

319class ObjectSizeStarSelectorTask(BaseSourceSelectorTask): 

320 r"""!A star selector that looks for a cluster of small objects in a size-magnitude plot 

321 

322 @anchor ObjectSizeStarSelectorTask_ 

323 

324 @section meas_algorithms_objectSizeStarSelector_Contents Contents 

325 

326 - @ref meas_algorithms_objectSizeStarSelector_Purpose 

327 - @ref meas_algorithms_objectSizeStarSelector_Initialize 

328 - @ref meas_algorithms_objectSizeStarSelector_IO 

329 - @ref meas_algorithms_objectSizeStarSelector_Config 

330 - @ref meas_algorithms_objectSizeStarSelector_Debug 

331 

332 @section meas_algorithms_objectSizeStarSelector_Purpose Description 

333 

334 A star selector that looks for a cluster of small objects in a size-magnitude plot. 

335 

336 @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation 

337 

338 @copydoc \_\_init\_\_ 

339 

340 @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task 

341 

342 Like all star selectors, the main method is `run`. 

343 

344 @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters 

345 

346 See @ref ObjectSizeStarSelectorConfig 

347 

348 @section meas_algorithms_objectSizeStarSelector_Debug Debug variables 

349 

350 ObjectSizeStarSelectorTask has a debug dictionary with the following keys: 

351 <dl> 

352 <dt>display 

353 <dd>bool; if True display debug information 

354 <dt>displayExposure 

355 <dd>bool; if True display the exposure and spatial cells 

356 <dt>plotMagSize 

357 <dd>bool: if True display the magnitude-size relation using matplotlib 

358 <dt>dumpData 

359 <dd>bool; if True dump data to a pickle file 

360 </dl> 

361 

362 For example, put something like: 

363 @code{.py} 

364 import lsstDebug 

365 def DebugInfo(name): 

366 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 

367 if name.endswith("objectSizeStarSelector"): 

368 di.display = True 

369 di.displayExposure = True 

370 di.plotMagSize = True 

371 

372 return di 

373 

374 lsstDebug.Info = DebugInfo 

375 @endcode 

376 into your `debug.py` file and run your task with the `--debug` flag. 

377 """ 

378 ConfigClass = ObjectSizeStarSelectorConfig 

379 usesMatches = False # selectStars does not use its matches argument 

380 

381 def selectSources(self, sourceCat, matches=None, exposure=None): 

382 """Return a selection of PSF candidates that represent likely stars. 

383 

384 A list of PSF candidates may be used by a PSF fitter to construct a PSF. 

385 

386 Parameters: 

387 ----------- 

388 sourceCat : `lsst.afw.table.SourceCatalog` 

389 Catalog of sources to select from. 

390 This catalog must be contiguous in memory. 

391 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

392 Ignored in this SourceSelector. 

393 exposure : `lsst.afw.image.Exposure` or None 

394 The exposure the catalog was built from; used to get the detector 

395 to transform to TanPix, and for debug display. 

396 

397 Return 

398 ------ 

399 struct : `lsst.pipe.base.Struct` 

400 The struct contains the following data: 

401 

402 - selected : `array` of `bool`` 

403 Boolean array of sources that were selected, same length as 

404 sourceCat. 

405 """ 

406 import lsstDebug 

407 display = lsstDebug.Info(__name__).display 

408 displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

409 plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation 

410 dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file? 

411 

412 detector = None 

413 pixToTanPix = None 

414 if exposure: 

415 detector = exposure.getDetector() 

416 if detector: 

417 pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS) 

418 # 

419 # Look at the distribution of stars in the magnitude-size plane 

420 # 

421 flux = sourceCat.get(self.config.sourceFluxField) 

422 fluxErr = sourceCat.get(self.config.sourceFluxField + "Err") 

423 

424 xx = numpy.empty(len(sourceCat)) 

425 xy = numpy.empty_like(xx) 

426 yy = numpy.empty_like(xx) 

427 for i, source in enumerate(sourceCat): 

428 Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy() 

429 if pixToTanPix: 

430 p = lsst.geom.Point2D(source.getX(), source.getY()) 

431 linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear() 

432 m = afwGeom.Quadrupole(Ixx, Iyy, Ixy) 

433 m.transform(linTransform) 

434 Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy() 

435 

436 xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy 

437 

438 width = numpy.sqrt(0.5*(xx + yy)) 

439 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

440 bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) 

441 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width))) 

442 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux))) 

443 if self.config.doFluxLimit: 

444 bad = numpy.logical_or(bad, flux < self.config.fluxMin) 

445 if self.config.fluxMax > 0: 

446 bad = numpy.logical_or(bad, flux > self.config.fluxMax) 

447 if self.config.doSignalToNoiseLimit: 

448 bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin) 

449 if self.config.signalToNoiseMax > 0: 449 ↛ 451line 449 didn't jump to line 451, because the condition on line 449 was never false

450 bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax) 

451 bad = numpy.logical_or(bad, width < self.config.widthMin) 

452 bad = numpy.logical_or(bad, width > self.config.widthMax) 

453 good = numpy.logical_not(bad) 

454 

455 if not numpy.any(good): 

456 raise RuntimeError("No objects passed our cuts for consideration as psf stars") 

457 

458 mag = -2.5*numpy.log10(flux[good]) 

459 width = width[good] 

460 # 

461 # Look for the maximum in the size histogram, then search upwards for the minimum that separates 

462 # the initial peak (of, we presume, stars) from the galaxies 

463 # 

464 if dumpData: 464 ↛ 465line 464 didn't jump to line 465, because the condition on line 464 was never true

465 import os 

466 import pickle as pickle 

467 _ii = 0 

468 while True: 

469 pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii)) 

470 if not os.path.exists(pickleFile): 

471 break 

472 _ii += 1 

473 

474 with open(pickleFile, "wb") as fd: 

475 pickle.dump(mag, fd, -1) 

476 pickle.dump(width, fd, -1) 

477 

478 centers, clusterId = _kcenters(width, nCluster=4, useMedian=True, 

479 widthStdAllowed=self.config.widthStdAllowed) 

480 

481 if display and plotMagSize: 481 ↛ 482line 481 didn't jump to line 482, because the condition on line 481 was never true

482 fig = plot(mag, width, centers, clusterId, 

483 magType=self.config.sourceFluxField.split(".")[-1].title(), 

484 marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True) 

485 else: 

486 fig = None 

487 

488 clusterId = _improveCluster(width, centers, clusterId, 

489 nsigma=self.config.nSigmaClip, 

490 widthStdAllowed=self.config.widthStdAllowed) 

491 

492 if display and plotMagSize: 492 ↛ 493line 492 didn't jump to line 493, because the condition on line 492 was never true

493 plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False) 

494 

495 stellar = (clusterId == 0) 

496 # 

497 # We know enough to plot, if so requested 

498 # 

499 frame = 0 

500 

501 if fig: 501 ↛ 502line 501 didn't jump to line 502, because the condition on line 501 was never true

502 if display and displayExposure: 

503 disp = afwDisplay.Display(frame=frame) 

504 disp.mtv(exposure.getMaskedImage(), title="PSF candidates") 

505 

506 global eventHandler 

507 eventHandler = EventHandler(fig.get_axes()[0], mag, width, 

508 sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame]) 

509 

510 fig.show() 

511 

512 while True: 

513 try: 

514 reply = input("continue? [c h(elp) q(uit) p(db)] ").strip() 

515 except EOFError: 

516 reply = None 

517 if not reply: 

518 reply = "c" 

519 

520 if reply: 

521 if reply[0] == "h": 

522 print("""\ 

523 We cluster the points; red are the stellar candidates and the other colours are other clusters. 

524 Points labelled + are rejects from the cluster (only for cluster 0). 

525 

526 At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text 

527 

528 If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the 

529 image display. 

530 """) 

531 elif reply[0] == "p": 

532 import pdb 

533 pdb.set_trace() 

534 elif reply[0] == 'q': 

535 sys.exit(1) 

536 else: 

537 break 

538 

539 if display and displayExposure: 539 ↛ 540line 539 didn't jump to line 540, because the condition on line 539 was never true

540 mi = exposure.getMaskedImage() 

541 with disp.Buffering(): 

542 for i, source in enumerate(sourceCat): 

543 if good[i]: 

544 ctype = afwDisplay.GREEN # star candidate 

545 else: 

546 ctype = afwDisplay.RED # not star 

547 

548 disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) 

549 

550 # stellar only applies to good==True objects 

551 mask = good == True # noqa (numpy bool comparison): E712 

552 good[mask] = stellar 

553 

554 return Struct(selected=good)