lsst.meas.algorithms  15.0-10-g113cadf7+2
objectSizeStarSelector.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 import sys
24 
25 import numpy
26 import warnings
27 from functools import reduce
28 
29 from lsst.afw.table import SourceCatalog
30 from lsst.log import Log
31 from lsst.pipe.base import Struct
32 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
33 import lsst.afw.geom as afwGeom
34 import lsst.pex.config as pexConfig
35 import lsst.afw.display.ds9 as ds9
36 from .starSelector import BaseStarSelectorTask, starSelectorRegistry
37 
38 
39 class ObjectSizeStarSelectorConfig(BaseStarSelectorTask.ConfigClass):
40  fluxMin = pexConfig.Field(
41  doc="specify the minimum psfFlux for good Psf Candidates",
42  dtype=float,
43  default=12500.0,
44  check=lambda x: x >= 0.0,
45  )
46  fluxMax = pexConfig.Field(
47  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
48  dtype=float,
49  default=0.0,
50  check=lambda x: x >= 0.0,
51  )
52  widthMin = pexConfig.Field(
53  doc="minimum width to include in histogram",
54  dtype=float,
55  default=0.0,
56  check=lambda x: x >= 0.0,
57  )
58  widthMax = pexConfig.Field(
59  doc="maximum width to include in histogram",
60  dtype=float,
61  default=10.0,
62  check=lambda x: x >= 0.0,
63  )
64  sourceFluxField = pexConfig.Field(
65  doc="Name of field in Source to use for flux measurement",
66  dtype=str,
67  default="base_GaussianFlux_flux",
68  )
69  widthStdAllowed = pexConfig.Field(
70  doc="Standard deviation of width allowed to be interpreted as good stars",
71  dtype=float,
72  default=0.15,
73  check=lambda x: x >= 0.0,
74  )
75  nSigmaClip = pexConfig.Field(
76  doc="Keep objects within this many sigma of cluster 0's median",
77  dtype=float,
78  default=2.0,
79  check=lambda x: x >= 0.0,
80  )
81 
82  def validate(self):
83  BaseStarSelectorTask.ConfigClass.validate(self)
84  if self.widthMin > self.widthMax:
85  raise pexConfig.FieldValidationError("widthMin (%f) > widthMax (%f)"
86  % (self.widthMin, self.widthMax))
87 
88 
90  """A class to handle key strokes with matplotlib displays"""
91 
92  def __init__(self, axes, xs, ys, x, y, frames=[0]):
93  self.axes = axes
94  self.xs = xs
95  self.ys = ys
96  self.x = x
97  self.y = y
98  self.frames = frames
99 
100  self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self)
101 
102  def __call__(self, ev):
103  if ev.inaxes != self.axes:
104  return
105 
106  if ev.key and ev.key in ("p"):
107  dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata)
108  dist[numpy.where(numpy.isnan(dist))] = 1e30
109 
110  which = numpy.where(dist == min(dist))
111 
112  x = self.x[which][0]
113  y = self.y[which][0]
114  for frame in self.frames:
115  ds9.pan(x, y, frame=frame)
116  ds9.cmdBuffer.flush()
117  else:
118  pass
119 
120 
121 def _assignClusters(yvec, centers):
122  """Return a vector of centerIds based on their distance to the centers"""
123  assert len(centers) > 0
124 
125  minDist = numpy.nan*numpy.ones_like(yvec)
126  clusterId = numpy.empty_like(yvec)
127  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
128  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
129  dbl.setLevel(dbl.INFO)
130 
131  # Make sure we are logging aall numpy warnings...
132  oldSettings = numpy.seterr(all="warn")
133  with warnings.catch_warnings(record=True) as w:
134  warnings.simplefilter("always")
135  for i, mean in enumerate(centers):
136  dist = abs(yvec - mean)
137  if i == 0:
138  update = dist == dist # True for all points
139  else:
140  update = dist < minDist
141  if w: # Only do if w is not empty i.e. contains a warning message
142  dbl.trace(str(w[-1]))
143 
144  minDist[update] = dist[update]
145  clusterId[update] = i
146  numpy.seterr(**oldSettings)
147 
148  return clusterId
149 
150 
151 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
152  """A classic k-means algorithm, clustering yvec into nCluster clusters
153 
154  Return the set of centres, and the cluster ID for each of the points
155 
156  If useMedian is true, use the median of the cluster as its centre, rather than
157  the traditional mean
158 
159  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
160  "e.g. why not use the Forgy or random partition initialization methods"
161  however, the approach adopted here seems to work well for the particular sorts of things
162  we're clustering in this application
163  """
164 
165  assert nCluster > 0
166 
167  mean0 = sorted(yvec)[len(yvec)//10] # guess
168  delta = mean0 * widthStdAllowed * 2.0
169  centers = mean0 + delta * numpy.arange(nCluster)
170 
171  func = numpy.median if useMedian else numpy.mean
172 
173  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
174  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
175  while True:
176  oclusterId = clusterId
177  clusterId = _assignClusters(yvec, centers)
178 
179  if numpy.all(clusterId == oclusterId):
180  break
181 
182  for i in range(nCluster):
183  # Only compute func if some points are available; otherwise, default to NaN.
184  pointsInCluster = (clusterId == i)
185  if numpy.any(pointsInCluster):
186  centers[i] = func(yvec[pointsInCluster])
187  else:
188  centers[i] = numpy.nan
189 
190  return centers, clusterId
191 
192 
193 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
194  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median"""
195 
196  nMember = sum(clusterId == clusterNum)
197  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
198  return clusterId
199  for iter in range(nIteration):
200  old_nMember = nMember
201 
202  inCluster0 = clusterId == clusterNum
203  yv = yvec[inCluster0]
204 
205  centers[clusterNum] = numpy.median(yv)
206  stdev = numpy.std(yv)
207 
208  syv = sorted(yv)
209  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
210  median = syv[int(0.5*nMember)]
211 
212  sd = stdev if stdev < stdev_iqr else stdev_iqr
213 
214  if False:
215  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
216  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
217  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
218  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
219 
220  nMember = sum(clusterId == clusterNum)
221  # 'sd < widthStdAllowed * median' prevents too much rejections
222  if nMember == old_nMember or sd < widthStdAllowed * median:
223  break
224 
225  return clusterId
226 
227 
228 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
229  magType="model", clear=True):
230 
231  log = Log.getLogger("objectSizeStarSelector.plot")
232  try:
233  import matplotlib.pyplot as plt
234  except ImportError as e:
235  log.warn("Unable to import matplotlib: %s", e)
236  return
237 
238  global fig
239  if not fig:
240  fig = plt.figure()
241  else:
242  if clear:
243  fig.clf()
244 
245  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
246 
247  xmin = sorted(mag)[int(0.05*len(mag))]
248  xmax = sorted(mag)[int(0.95*len(mag))]
249 
250  axes.set_xlim(-17.5, -13)
251  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
252  axes.set_ylim(0, 10)
253 
254  colors = ["r", "g", "b", "c", "m", "k", ]
255  for k, mean in enumerate(centers):
256  if k == 0:
257  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
258 
259  li = (clusterId == k)
260  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
261  color=colors[k % len(colors)])
262 
263  li = (clusterId == -1)
264  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
265  color='k')
266 
267  if clear:
268  axes.set_xlabel("Instrumental %s mag" % magType)
269  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
270 
271  return fig
272 
273 
279 
280 
282  """!A star selector that looks for a cluster of small objects in a size-magnitude plot
283 
284  @anchor ObjectSizeStarSelectorTask_
285 
286  @section meas_algorithms_objectSizeStarSelector_Contents Contents
287 
288  - @ref meas_algorithms_objectSizeStarSelector_Purpose
289  - @ref meas_algorithms_objectSizeStarSelector_Initialize
290  - @ref meas_algorithms_objectSizeStarSelector_IO
291  - @ref meas_algorithms_objectSizeStarSelector_Config
292  - @ref meas_algorithms_objectSizeStarSelector_Debug
293 
294  @section meas_algorithms_objectSizeStarSelector_Purpose Description
295 
296  A star selector that looks for a cluster of small objects in a size-magnitude plot.
297 
298  @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation
299 
300  @copydoc \_\_init\_\_
301 
302  @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task
303 
304  Like all star selectors, the main method is `run`.
305 
306  @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters
307 
308  See @ref ObjectSizeStarSelectorConfig
309 
310  @section meas_algorithms_objectSizeStarSelector_Debug Debug variables
311 
312  ObjectSizeStarSelectorTask has a debug dictionary with the following keys:
313  <dl>
314  <dt>display
315  <dd>bool; if True display debug information
316  <dt>displayExposure
317  <dd>bool; if True display the exposure and spatial cells
318  <dt>plotMagSize
319  <dd>bool: if True display the magnitude-size relation using matplotlib
320  <dt>dumpData
321  <dd>bool; if True dump data to a pickle file
322  </dl>
323 
324  For example, put something like:
325  @code{.py}
326  import lsstDebug
327  def DebugInfo(name):
328  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
329  if name.endswith("objectSizeStarSelector"):
330  di.display = True
331  di.displayExposure = True
332  di.plotMagSize = True
333 
334  return di
335 
336  lsstDebug.Info = DebugInfo
337  @endcode
338  into your `debug.py` file and run your task with the `--debug` flag.
339  """
340  ConfigClass = ObjectSizeStarSelectorConfig
341  usesMatches = False # selectStars does not use its matches argument
342 
343  def selectStars(self, exposure, sourceCat, matches=None):
344  """!Return a list of PSF candidates that represent likely stars
345 
346  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
347 
348  @param[in] exposure the exposure containing the sources
349  @param[in] sourceCat catalog of sources that may be stars (an lsst.afw.table.SourceCatalog)
350  @param[in] matches astrometric matches; ignored by this star selector
351 
352  @return an lsst.pipe.base.Struct containing:
353  - starCat catalog of selected stars (a subset of sourceCat)
354  """
355  import lsstDebug
356  display = lsstDebug.Info(__name__).display
357  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
358  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
359  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
360 
361  detector = exposure.getDetector()
362  pixToTanPix = None
363  if detector is not None:
364  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
365  #
366  # Look at the distribution of stars in the magnitude-size plane
367  #
368  flux = sourceCat.get(self.config.sourceFluxField)
369 
370  xx = numpy.empty(len(sourceCat))
371  xy = numpy.empty_like(xx)
372  yy = numpy.empty_like(xx)
373  for i, source in enumerate(sourceCat):
374  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
375  if pixToTanPix:
376  p = afwGeom.Point2D(source.getX(), source.getY())
377  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
378  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
379  m.transform(linTransform)
380  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
381 
382  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
383 
384  width = numpy.sqrt(0.5*(xx + yy))
385  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
386  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
387  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
388  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
389  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
390  bad = numpy.logical_or(bad, width < self.config.widthMin)
391  bad = numpy.logical_or(bad, width > self.config.widthMax)
392  if self.config.fluxMax > 0:
393  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
394  good = numpy.logical_not(bad)
395 
396  if not numpy.any(good):
397  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
398 
399  mag = -2.5*numpy.log10(flux[good])
400  width = width[good]
401  #
402  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
403  # the initial peak (of, we presume, stars) from the galaxies
404  #
405  if dumpData:
406  import os
407  import pickle as pickle
408  _ii = 0
409  while True:
410  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
411  if not os.path.exists(pickleFile):
412  break
413  _ii += 1
414 
415  with open(pickleFile, "wb") as fd:
416  pickle.dump(mag, fd, -1)
417  pickle.dump(width, fd, -1)
418 
419  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
420  widthStdAllowed=self.config.widthStdAllowed)
421 
422  if display and plotMagSize:
423  fig = plot(mag, width, centers, clusterId,
424  magType=self.config.sourceFluxField.split(".")[-1].title(),
425  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
426  else:
427  fig = None
428 
429  clusterId = _improveCluster(width, centers, clusterId,
430  nsigma=self.config.nSigmaClip,
431  widthStdAllowed=self.config.widthStdAllowed)
432 
433  if display and plotMagSize:
434  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
435 
436  stellar = (clusterId == 0)
437  #
438  # We know enough to plot, if so requested
439  #
440  frame = 0
441 
442  if fig:
443  if display and displayExposure:
444  ds9.mtv(exposure.getMaskedImage(), frame=frame, title="PSF candidates")
445 
446  global eventHandler
447  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
448  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
449 
450  fig.show()
451 
452  while True:
453  try:
454  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
455  except EOFError:
456  reply = None
457  if not reply:
458  reply = "c"
459 
460  if reply:
461  if reply[0] == "h":
462  print("""\
463  We cluster the points; red are the stellar candidates and the other colours are other clusters.
464  Points labelled + are rejects from the cluster (only for cluster 0).
465 
466  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
467 
468  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
469  """)
470  elif reply[0] == "p":
471  import pdb
472  pdb.set_trace()
473  elif reply[0] == 'q':
474  sys.exit(1)
475  else:
476  break
477 
478  if display and displayExposure:
479  mi = exposure.getMaskedImage()
480 
481  with ds9.Buffering():
482  for i, source in enumerate(sourceCat):
483  if good[i]:
484  ctype = ds9.GREEN # star candidate
485  else:
486  ctype = ds9.RED # not star
487 
488  ds9.dot("+", source.getX() - mi.getX0(),
489  source.getY() - mi.getY0(), frame=frame, ctype=ctype)
490 
491  starCat = SourceCatalog(sourceCat.table)
492  goodSources = [s for g, s in zip(good, sourceCat) if g]
493  for isStellar, source in zip(stellar, goodSources):
494  if isStellar:
495  starCat.append(source)
496 
497  return Struct(
498  starCat=starCat,
499  )
500 
501 
502 starSelectorRegistry.register("objectSize", ObjectSizeStarSelectorTask)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)
A star selector that looks for a cluster of small objects in a size-magnitude plot.
AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, Point2D const &inPoint)
def selectStars(self, exposure, sourceCat, matches=None)
Return a list of PSF candidates that represent likely stars.