lsst.meas.algorithms  13.0-24-g22030a45+14
objectSizeStarSelector.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import print_function
24 from future import standard_library
25 standard_library.install_aliases()
26 from builtins import zip
27 from builtins import input
28 from builtins import str
29 from builtins import range
30 from builtins import object
31 import sys
32 
33 import numpy
34 import warnings
35 from functools import reduce
36 
37 from lsst.afw.table import SourceCatalog
38 from lsst.log import Log
39 from lsst.pipe.base import Struct
40 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
41 from lsst.afw.geom.ellipses import Quadrupole
42 import lsst.afw.geom as afwGeom
43 import lsst.pex.config as pexConfig
44 import lsst.afw.display.ds9 as ds9
45 from .starSelector import BaseStarSelectorTask, starSelectorRegistry
46 
47 
48 class ObjectSizeStarSelectorConfig(BaseStarSelectorTask.ConfigClass):
49  fluxMin = pexConfig.Field(
50  doc="specify the minimum psfFlux for good Psf Candidates",
51  dtype=float,
52  default=12500.0,
53  check=lambda x: x >= 0.0,
54  )
55  fluxMax = pexConfig.Field(
56  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
57  dtype=float,
58  default=0.0,
59  check=lambda x: x >= 0.0,
60  )
61  widthMin = pexConfig.Field(
62  doc="minimum width to include in histogram",
63  dtype=float,
64  default=0.0,
65  check=lambda x: x >= 0.0,
66  )
67  widthMax = pexConfig.Field(
68  doc="maximum width to include in histogram",
69  dtype=float,
70  default=10.0,
71  check=lambda x: x >= 0.0,
72  )
73  sourceFluxField = pexConfig.Field(
74  doc="Name of field in Source to use for flux measurement",
75  dtype=str,
76  default="base_GaussianFlux_flux",
77  )
78  widthStdAllowed = pexConfig.Field(
79  doc="Standard deviation of width allowed to be interpreted as good stars",
80  dtype=float,
81  default=0.15,
82  check=lambda x: x >= 0.0,
83  )
84  nSigmaClip = pexConfig.Field(
85  doc="Keep objects within this many sigma of cluster 0's median",
86  dtype=float,
87  default=2.0,
88  check=lambda x: x >= 0.0,
89  )
90 
91  def validate(self):
92  BaseStarSelectorTask.ConfigClass.validate(self)
93  if self.widthMin > self.widthMax:
94  raise pexConfig.FieldValidationError("widthMin (%f) > widthMax (%f)"
95  % (self.widthMin, self.widthMax))
96 
97 
98 class EventHandler(object):
99  """A class to handle key strokes with matplotlib displays"""
100 
101  def __init__(self, axes, xs, ys, x, y, frames=[0]):
102  self.axes = axes
103  self.xs = xs
104  self.ys = ys
105  self.x = x
106  self.y = y
107  self.frames = frames
108 
109  self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self)
110 
111  def __call__(self, ev):
112  if ev.inaxes != self.axes:
113  return
114 
115  if ev.key and ev.key in ("p"):
116  dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata)
117  dist[numpy.where(numpy.isnan(dist))] = 1e30
118 
119  which = numpy.where(dist == min(dist))
120 
121  x = self.x[which][0]
122  y = self.y[which][0]
123  for frame in self.frames:
124  ds9.pan(x, y, frame=frame)
125  ds9.cmdBuffer.flush()
126  else:
127  pass
128 
129 
130 def _assignClusters(yvec, centers):
131  """Return a vector of centerIds based on their distance to the centers"""
132  assert len(centers) > 0
133 
134  minDist = numpy.nan*numpy.ones_like(yvec)
135  clusterId = numpy.empty_like(yvec)
136  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
137  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
138  dbl.setLevel(dbl.INFO)
139 
140  # Make sure we are logging aall numpy warnings...
141  oldSettings = numpy.seterr(all="warn")
142  with warnings.catch_warnings(record=True) as w:
143  warnings.simplefilter("always")
144  for i, mean in enumerate(centers):
145  dist = abs(yvec - mean)
146  if i == 0:
147  update = dist == dist # True for all points
148  else:
149  update = dist < minDist
150  if w: # Only do if w is not empty i.e. contains a warning message
151  dbl.trace(str(w[-1]))
152 
153  minDist[update] = dist[update]
154  clusterId[update] = i
155  numpy.seterr(**oldSettings)
156 
157  return clusterId
158 
159 
160 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
161  """A classic k-means algorithm, clustering yvec into nCluster clusters
162 
163  Return the set of centres, and the cluster ID for each of the points
164 
165  If useMedian is true, use the median of the cluster as its centre, rather than
166  the traditional mean
167 
168  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
169  "e.g. why not use the Forgy or random partition initialization methods"
170  however, the approach adopted here seems to work well for the particular sorts of things
171  we're clustering in this application
172  """
173 
174  assert nCluster > 0
175 
176  mean0 = sorted(yvec)[len(yvec)//10] # guess
177  delta = mean0 * widthStdAllowed * 2.0
178  centers = mean0 + delta * numpy.arange(nCluster)
179 
180  func = numpy.median if useMedian else numpy.mean
181 
182  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
183  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
184  while True:
185  oclusterId = clusterId
186  clusterId = _assignClusters(yvec, centers)
187 
188  if numpy.all(clusterId == oclusterId):
189  break
190 
191  for i in range(nCluster):
192  # Only compute func if some points are available; otherwise, default to NaN.
193  pointsInCluster = (clusterId == i)
194  if numpy.any(pointsInCluster):
195  centers[i] = func(yvec[pointsInCluster])
196  else:
197  centers[i] = numpy.nan
198 
199  return centers, clusterId
200 
201 
202 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
203  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median"""
204 
205  nMember = sum(clusterId == clusterNum)
206  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
207  return clusterId
208  for iter in range(nIteration):
209  old_nMember = nMember
210 
211  inCluster0 = clusterId == clusterNum
212  yv = yvec[inCluster0]
213 
214  centers[clusterNum] = numpy.median(yv)
215  stdev = numpy.std(yv)
216 
217  syv = sorted(yv)
218  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
219  median = syv[int(0.5*nMember)]
220 
221  sd = stdev if stdev < stdev_iqr else stdev_iqr
222 
223  if False:
224  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
225  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
226  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
227  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
228 
229  nMember = sum(clusterId == clusterNum)
230  # 'sd < widthStdAllowed * median' prevents too much rejections
231  if nMember == old_nMember or sd < widthStdAllowed * median:
232  break
233 
234  return clusterId
235 
236 
237 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
238  magType="model", clear=True):
239 
240  log = Log.getLogger("objectSizeStarSelector.plot")
241  try:
242  import matplotlib.pyplot as plt
243  except ImportError as e:
244  log.warn("Unable to import matplotlib: %s", e)
245  return
246 
247  global fig
248  if not fig:
249  fig = plt.figure()
250  else:
251  if clear:
252  fig.clf()
253 
254  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
255 
256  xmin = sorted(mag)[int(0.05*len(mag))]
257  xmax = sorted(mag)[int(0.95*len(mag))]
258 
259  axes.set_xlim(-17.5, -13)
260  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
261  axes.set_ylim(0, 10)
262 
263  colors = ["r", "g", "b", "c", "m", "k", ]
264  for k, mean in enumerate(centers):
265  if k == 0:
266  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
267 
268  l = (clusterId == k)
269  axes.plot(mag[l], width[l], marker, markersize=markersize, markeredgewidth=markeredgewidth,
270  color=colors[k % len(colors)])
271 
272  l = (clusterId == -1)
273  axes.plot(mag[l], width[l], marker, markersize=markersize, markeredgewidth=markeredgewidth,
274  color='k')
275 
276  if clear:
277  axes.set_xlabel("Instrumental %s mag" % magType)
278  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
279 
280  return fig
281 
282 
288 
289 
291  """!A star selector that looks for a cluster of small objects in a size-magnitude plot
292 
293  @anchor ObjectSizeStarSelectorTask_
294 
295  @section meas_algorithms_objectSizeStarSelector_Contents Contents
296 
297  - @ref meas_algorithms_objectSizeStarSelector_Purpose
298  - @ref meas_algorithms_objectSizeStarSelector_Initialize
299  - @ref meas_algorithms_objectSizeStarSelector_IO
300  - @ref meas_algorithms_objectSizeStarSelector_Config
301  - @ref meas_algorithms_objectSizeStarSelector_Debug
302 
303  @section meas_algorithms_objectSizeStarSelector_Purpose Description
304 
305  A star selector that looks for a cluster of small objects in a size-magnitude plot.
306 
307  @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation
308 
309  @copydoc \_\_init\_\_
310 
311  @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task
312 
313  Like all star selectors, the main method is `run`.
314 
315  @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters
316 
317  See @ref ObjectSizeStarSelectorConfig
318 
319  @section meas_algorithms_objectSizeStarSelector_Debug Debug variables
320 
321  ObjectSizeStarSelectorTask has a debug dictionary with the following keys:
322  <dl>
323  <dt>display
324  <dd>bool; if True display debug information
325  <dt>displayExposure
326  <dd>bool; if True display the exposure and spatial cells
327  <dt>plotMagSize
328  <dd>bool: if True display the magnitude-size relation using matplotlib
329  <dt>dumpData
330  <dd>bool; if True dump data to a pickle file
331  </dl>
332 
333  For example, put something like:
334  @code{.py}
335  import lsstDebug
336  def DebugInfo(name):
337  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
338  if name.endswith("objectSizeStarSelector"):
339  di.display = True
340  di.displayExposure = True
341  di.plotMagSize = True
342 
343  return di
344 
345  lsstDebug.Info = DebugInfo
346  @endcode
347  into your `debug.py` file and run your task with the `--debug` flag.
348  """
349  ConfigClass = ObjectSizeStarSelectorConfig
350  usesMatches = False # selectStars does not use its matches argument
351 
352  def selectStars(self, exposure, sourceCat, matches=None):
353  """!Return a list of PSF candidates that represent likely stars
354 
355  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
356 
357  \param[in] exposure the exposure containing the sources
358  \param[in] sourceCat catalog of sources that may be stars (an lsst.afw.table.SourceCatalog)
359  \param[in] matches astrometric matches; ignored by this star selector
360 
361  \return an lsst.pipe.base.Struct containing:
362  - starCat catalog of selected stars (a subset of sourceCat)
363  """
364  import lsstDebug
365  display = lsstDebug.Info(__name__).display
366  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
367  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
368  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
369 
370  detector = exposure.getDetector()
371  pixToTanPix = None
372  if detector is not None:
373  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
374  #
375  # Look at the distribution of stars in the magnitude-size plane
376  #
377  flux = sourceCat.get(self.config.sourceFluxField)
378 
379  xx = numpy.empty(len(sourceCat))
380  xy = numpy.empty_like(xx)
381  yy = numpy.empty_like(xx)
382  for i, source in enumerate(sourceCat):
383  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
384  if pixToTanPix:
385  p = afwGeom.Point2D(source.getX(), source.getY())
386  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
387  m = Quadrupole(Ixx, Iyy, Ixy)
388  m.transform(linTransform)
389  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
390 
391  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
392 
393  width = numpy.sqrt(0.5*(xx + yy))
394 
395  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
396  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
397  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
398  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
399  bad = numpy.logical_or(bad, width < self.config.widthMin)
400  bad = numpy.logical_or(bad, width > self.config.widthMax)
401  if self.config.fluxMax > 0:
402  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
403  good = numpy.logical_not(bad)
404 
405  if not numpy.any(good):
406  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
407 
408  mag = -2.5*numpy.log10(flux[good])
409  width = width[good]
410  #
411  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
412  # the initial peak (of, we presume, stars) from the galaxies
413  #
414  if dumpData:
415  import os
416  import pickle as pickle
417  _ii = 0
418  while True:
419  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
420  if not os.path.exists(pickleFile):
421  break
422  _ii += 1
423 
424  with open(pickleFile, "wb") as fd:
425  pickle.dump(mag, fd, -1)
426  pickle.dump(width, fd, -1)
427 
428  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
429  widthStdAllowed=self.config.widthStdAllowed)
430 
431  if display and plotMagSize:
432  fig = plot(mag, width, centers, clusterId,
433  magType=self.config.sourceFluxField.split(".")[-1].title(),
434  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
435  else:
436  fig = None
437 
438  clusterId = _improveCluster(width, centers, clusterId,
439  nsigma=self.config.nSigmaClip,
440  widthStdAllowed=self.config.widthStdAllowed)
441 
442  if display and plotMagSize:
443  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
444 
445  stellar = (clusterId == 0)
446  #
447  # We know enough to plot, if so requested
448  #
449  frame = 0
450 
451  if fig:
452  if display and displayExposure:
453  ds9.mtv(exposure.getMaskedImage(), frame=frame, title="PSF candidates")
454 
455  global eventHandler
456  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
457  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
458 
459  fig.show()
460 
461  while True:
462  try:
463  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
464  except EOFError:
465  reply = None
466  if not reply:
467  reply = "c"
468 
469  if reply:
470  if reply[0] == "h":
471  print("""\
472  We cluster the points; red are the stellar candidates and the other colours are other clusters.
473  Points labelled + are rejects from the cluster (only for cluster 0).
474 
475  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
476 
477  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
478  """)
479  elif reply[0] == "p":
480  import pdb
481  pdb.set_trace()
482  elif reply[0] == 'q':
483  sys.exit(1)
484  else:
485  break
486 
487  if display and displayExposure:
488  mi = exposure.getMaskedImage()
489 
490  with ds9.Buffering():
491  for i, source in enumerate(sourceCat):
492  if good[i]:
493  ctype = ds9.GREEN # star candidate
494  else:
495  ctype = ds9.RED # not star
496 
497  ds9.dot("+", source.getX() - mi.getX0(),
498  source.getY() - mi.getY0(), frame=frame, ctype=ctype)
499 
500  starCat = SourceCatalog(sourceCat.table)
501  goodSources = [s for g, s in zip(good, sourceCat) if g]
502  for isStellar, source in zip(stellar, goodSources):
503  if isStellar:
504  starCat.append(source)
505 
506  return Struct(
507  starCat=starCat,
508  )
509 
510 
511 starSelectorRegistry.register("objectSize", ObjectSizeStarSelectorTask)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)
A star selector that looks for a cluster of small objects in a size-magnitude plot.
AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, Point2D const &inPoint)
def selectStars(self, exposure, sourceCat, matches=None)
Return a list of PSF candidates that represent likely stars.