lsst.meas.algorithms  14.0-19-gbbfecc2c+1
objectSizeStarSelector.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import print_function
24 from future import standard_library
25 standard_library.install_aliases()
26 from builtins import zip
27 from builtins import input
28 from builtins import str
29 from builtins import range
30 from builtins import object
31 import sys
32 
33 import numpy
34 import warnings
35 from functools import reduce
36 
37 from lsst.afw.table import SourceCatalog
38 from lsst.log import Log
39 from lsst.pipe.base import Struct
40 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
41 import lsst.afw.geom as afwGeom
42 import lsst.pex.config as pexConfig
43 import lsst.afw.display.ds9 as ds9
44 from .starSelector import BaseStarSelectorTask, starSelectorRegistry
45 
46 
47 class ObjectSizeStarSelectorConfig(BaseStarSelectorTask.ConfigClass):
48  fluxMin = pexConfig.Field(
49  doc="specify the minimum psfFlux for good Psf Candidates",
50  dtype=float,
51  default=12500.0,
52  check=lambda x: x >= 0.0,
53  )
54  fluxMax = pexConfig.Field(
55  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
56  dtype=float,
57  default=0.0,
58  check=lambda x: x >= 0.0,
59  )
60  widthMin = pexConfig.Field(
61  doc="minimum width to include in histogram",
62  dtype=float,
63  default=0.0,
64  check=lambda x: x >= 0.0,
65  )
66  widthMax = pexConfig.Field(
67  doc="maximum width to include in histogram",
68  dtype=float,
69  default=10.0,
70  check=lambda x: x >= 0.0,
71  )
72  sourceFluxField = pexConfig.Field(
73  doc="Name of field in Source to use for flux measurement",
74  dtype=str,
75  default="base_GaussianFlux_flux",
76  )
77  widthStdAllowed = pexConfig.Field(
78  doc="Standard deviation of width allowed to be interpreted as good stars",
79  dtype=float,
80  default=0.15,
81  check=lambda x: x >= 0.0,
82  )
83  nSigmaClip = pexConfig.Field(
84  doc="Keep objects within this many sigma of cluster 0's median",
85  dtype=float,
86  default=2.0,
87  check=lambda x: x >= 0.0,
88  )
89 
90  def validate(self):
91  BaseStarSelectorTask.ConfigClass.validate(self)
92  if self.widthMin > self.widthMax:
93  raise pexConfig.FieldValidationError("widthMin (%f) > widthMax (%f)"
94  % (self.widthMin, self.widthMax))
95 
96 
97 class EventHandler(object):
98  """A class to handle key strokes with matplotlib displays"""
99 
100  def __init__(self, axes, xs, ys, x, y, frames=[0]):
101  self.axes = axes
102  self.xs = xs
103  self.ys = ys
104  self.x = x
105  self.y = y
106  self.frames = frames
107 
108  self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self)
109 
110  def __call__(self, ev):
111  if ev.inaxes != self.axes:
112  return
113 
114  if ev.key and ev.key in ("p"):
115  dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata)
116  dist[numpy.where(numpy.isnan(dist))] = 1e30
117 
118  which = numpy.where(dist == min(dist))
119 
120  x = self.x[which][0]
121  y = self.y[which][0]
122  for frame in self.frames:
123  ds9.pan(x, y, frame=frame)
124  ds9.cmdBuffer.flush()
125  else:
126  pass
127 
128 
129 def _assignClusters(yvec, centers):
130  """Return a vector of centerIds based on their distance to the centers"""
131  assert len(centers) > 0
132 
133  minDist = numpy.nan*numpy.ones_like(yvec)
134  clusterId = numpy.empty_like(yvec)
135  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
136  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
137  dbl.setLevel(dbl.INFO)
138 
139  # Make sure we are logging aall numpy warnings...
140  oldSettings = numpy.seterr(all="warn")
141  with warnings.catch_warnings(record=True) as w:
142  warnings.simplefilter("always")
143  for i, mean in enumerate(centers):
144  dist = abs(yvec - mean)
145  if i == 0:
146  update = dist == dist # True for all points
147  else:
148  update = dist < minDist
149  if w: # Only do if w is not empty i.e. contains a warning message
150  dbl.trace(str(w[-1]))
151 
152  minDist[update] = dist[update]
153  clusterId[update] = i
154  numpy.seterr(**oldSettings)
155 
156  return clusterId
157 
158 
159 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
160  """A classic k-means algorithm, clustering yvec into nCluster clusters
161 
162  Return the set of centres, and the cluster ID for each of the points
163 
164  If useMedian is true, use the median of the cluster as its centre, rather than
165  the traditional mean
166 
167  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
168  "e.g. why not use the Forgy or random partition initialization methods"
169  however, the approach adopted here seems to work well for the particular sorts of things
170  we're clustering in this application
171  """
172 
173  assert nCluster > 0
174 
175  mean0 = sorted(yvec)[len(yvec)//10] # guess
176  delta = mean0 * widthStdAllowed * 2.0
177  centers = mean0 + delta * numpy.arange(nCluster)
178 
179  func = numpy.median if useMedian else numpy.mean
180 
181  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
182  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
183  while True:
184  oclusterId = clusterId
185  clusterId = _assignClusters(yvec, centers)
186 
187  if numpy.all(clusterId == oclusterId):
188  break
189 
190  for i in range(nCluster):
191  # Only compute func if some points are available; otherwise, default to NaN.
192  pointsInCluster = (clusterId == i)
193  if numpy.any(pointsInCluster):
194  centers[i] = func(yvec[pointsInCluster])
195  else:
196  centers[i] = numpy.nan
197 
198  return centers, clusterId
199 
200 
201 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
202  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median"""
203 
204  nMember = sum(clusterId == clusterNum)
205  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
206  return clusterId
207  for iter in range(nIteration):
208  old_nMember = nMember
209 
210  inCluster0 = clusterId == clusterNum
211  yv = yvec[inCluster0]
212 
213  centers[clusterNum] = numpy.median(yv)
214  stdev = numpy.std(yv)
215 
216  syv = sorted(yv)
217  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
218  median = syv[int(0.5*nMember)]
219 
220  sd = stdev if stdev < stdev_iqr else stdev_iqr
221 
222  if False:
223  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
224  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
225  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
226  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
227 
228  nMember = sum(clusterId == clusterNum)
229  # 'sd < widthStdAllowed * median' prevents too much rejections
230  if nMember == old_nMember or sd < widthStdAllowed * median:
231  break
232 
233  return clusterId
234 
235 
236 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
237  magType="model", clear=True):
238 
239  log = Log.getLogger("objectSizeStarSelector.plot")
240  try:
241  import matplotlib.pyplot as plt
242  except ImportError as e:
243  log.warn("Unable to import matplotlib: %s", e)
244  return
245 
246  global fig
247  if not fig:
248  fig = plt.figure()
249  else:
250  if clear:
251  fig.clf()
252 
253  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
254 
255  xmin = sorted(mag)[int(0.05*len(mag))]
256  xmax = sorted(mag)[int(0.95*len(mag))]
257 
258  axes.set_xlim(-17.5, -13)
259  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
260  axes.set_ylim(0, 10)
261 
262  colors = ["r", "g", "b", "c", "m", "k", ]
263  for k, mean in enumerate(centers):
264  if k == 0:
265  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
266 
267  l = (clusterId == k)
268  axes.plot(mag[l], width[l], marker, markersize=markersize, markeredgewidth=markeredgewidth,
269  color=colors[k % len(colors)])
270 
271  l = (clusterId == -1)
272  axes.plot(mag[l], width[l], marker, markersize=markersize, markeredgewidth=markeredgewidth,
273  color='k')
274 
275  if clear:
276  axes.set_xlabel("Instrumental %s mag" % magType)
277  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
278 
279  return fig
280 
281 
287 
288 
290  """!A star selector that looks for a cluster of small objects in a size-magnitude plot
291 
292  @anchor ObjectSizeStarSelectorTask_
293 
294  @section meas_algorithms_objectSizeStarSelector_Contents Contents
295 
296  - @ref meas_algorithms_objectSizeStarSelector_Purpose
297  - @ref meas_algorithms_objectSizeStarSelector_Initialize
298  - @ref meas_algorithms_objectSizeStarSelector_IO
299  - @ref meas_algorithms_objectSizeStarSelector_Config
300  - @ref meas_algorithms_objectSizeStarSelector_Debug
301 
302  @section meas_algorithms_objectSizeStarSelector_Purpose Description
303 
304  A star selector that looks for a cluster of small objects in a size-magnitude plot.
305 
306  @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation
307 
308  @copydoc \_\_init\_\_
309 
310  @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task
311 
312  Like all star selectors, the main method is `run`.
313 
314  @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters
315 
316  See @ref ObjectSizeStarSelectorConfig
317 
318  @section meas_algorithms_objectSizeStarSelector_Debug Debug variables
319 
320  ObjectSizeStarSelectorTask has a debug dictionary with the following keys:
321  <dl>
322  <dt>display
323  <dd>bool; if True display debug information
324  <dt>displayExposure
325  <dd>bool; if True display the exposure and spatial cells
326  <dt>plotMagSize
327  <dd>bool: if True display the magnitude-size relation using matplotlib
328  <dt>dumpData
329  <dd>bool; if True dump data to a pickle file
330  </dl>
331 
332  For example, put something like:
333  @code{.py}
334  import lsstDebug
335  def DebugInfo(name):
336  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
337  if name.endswith("objectSizeStarSelector"):
338  di.display = True
339  di.displayExposure = True
340  di.plotMagSize = True
341 
342  return di
343 
344  lsstDebug.Info = DebugInfo
345  @endcode
346  into your `debug.py` file and run your task with the `--debug` flag.
347  """
348  ConfigClass = ObjectSizeStarSelectorConfig
349  usesMatches = False # selectStars does not use its matches argument
350 
351  def selectStars(self, exposure, sourceCat, matches=None):
352  """!Return a list of PSF candidates that represent likely stars
353 
354  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
355 
356  \param[in] exposure the exposure containing the sources
357  \param[in] sourceCat catalog of sources that may be stars (an lsst.afw.table.SourceCatalog)
358  \param[in] matches astrometric matches; ignored by this star selector
359 
360  \return an lsst.pipe.base.Struct containing:
361  - starCat catalog of selected stars (a subset of sourceCat)
362  """
363  import lsstDebug
364  display = lsstDebug.Info(__name__).display
365  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
366  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
367  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
368 
369  detector = exposure.getDetector()
370  pixToTanPix = None
371  if detector is not None:
372  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
373  #
374  # Look at the distribution of stars in the magnitude-size plane
375  #
376  flux = sourceCat.get(self.config.sourceFluxField)
377 
378  xx = numpy.empty(len(sourceCat))
379  xy = numpy.empty_like(xx)
380  yy = numpy.empty_like(xx)
381  for i, source in enumerate(sourceCat):
382  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
383  if pixToTanPix:
384  p = afwGeom.Point2D(source.getX(), source.getY())
385  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
386  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
387  m.transform(linTransform)
388  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
389 
390  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
391 
392  width = numpy.sqrt(0.5*(xx + yy))
393  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
394  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
395  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
396  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
397  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
398  bad = numpy.logical_or(bad, width < self.config.widthMin)
399  bad = numpy.logical_or(bad, width > self.config.widthMax)
400  if self.config.fluxMax > 0:
401  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
402  good = numpy.logical_not(bad)
403 
404  if not numpy.any(good):
405  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
406 
407  mag = -2.5*numpy.log10(flux[good])
408  width = width[good]
409  #
410  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
411  # the initial peak (of, we presume, stars) from the galaxies
412  #
413  if dumpData:
414  import os
415  import pickle as pickle
416  _ii = 0
417  while True:
418  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
419  if not os.path.exists(pickleFile):
420  break
421  _ii += 1
422 
423  with open(pickleFile, "wb") as fd:
424  pickle.dump(mag, fd, -1)
425  pickle.dump(width, fd, -1)
426 
427  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
428  widthStdAllowed=self.config.widthStdAllowed)
429 
430  if display and plotMagSize:
431  fig = plot(mag, width, centers, clusterId,
432  magType=self.config.sourceFluxField.split(".")[-1].title(),
433  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
434  else:
435  fig = None
436 
437  clusterId = _improveCluster(width, centers, clusterId,
438  nsigma=self.config.nSigmaClip,
439  widthStdAllowed=self.config.widthStdAllowed)
440 
441  if display and plotMagSize:
442  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
443 
444  stellar = (clusterId == 0)
445  #
446  # We know enough to plot, if so requested
447  #
448  frame = 0
449 
450  if fig:
451  if display and displayExposure:
452  ds9.mtv(exposure.getMaskedImage(), frame=frame, title="PSF candidates")
453 
454  global eventHandler
455  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
456  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
457 
458  fig.show()
459 
460  while True:
461  try:
462  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
463  except EOFError:
464  reply = None
465  if not reply:
466  reply = "c"
467 
468  if reply:
469  if reply[0] == "h":
470  print("""\
471  We cluster the points; red are the stellar candidates and the other colours are other clusters.
472  Points labelled + are rejects from the cluster (only for cluster 0).
473 
474  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
475 
476  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
477  """)
478  elif reply[0] == "p":
479  import pdb
480  pdb.set_trace()
481  elif reply[0] == 'q':
482  sys.exit(1)
483  else:
484  break
485 
486  if display and displayExposure:
487  mi = exposure.getMaskedImage()
488 
489  with ds9.Buffering():
490  for i, source in enumerate(sourceCat):
491  if good[i]:
492  ctype = ds9.GREEN # star candidate
493  else:
494  ctype = ds9.RED # not star
495 
496  ds9.dot("+", source.getX() - mi.getX0(),
497  source.getY() - mi.getY0(), frame=frame, ctype=ctype)
498 
499  starCat = SourceCatalog(sourceCat.table)
500  goodSources = [s for g, s in zip(good, sourceCat) if g]
501  for isStellar, source in zip(stellar, goodSources):
502  if isStellar:
503  starCat.append(source)
504 
505  return Struct(
506  starCat=starCat,
507  )
508 
509 
510 starSelectorRegistry.register("objectSize", ObjectSizeStarSelectorTask)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)
A star selector that looks for a cluster of small objects in a size-magnitude plot.
AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, Point2D const &inPoint)
def selectStars(self, exposure, sourceCat, matches=None)
Return a list of PSF candidates that represent likely stars.