lsst.meas.algorithms  13.0-16-g6e7f056
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
objectSizeStarSelector.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import print_function
24 from future import standard_library
25 standard_library.install_aliases()
26 from builtins import zip
27 from builtins import input
28 from builtins import str
29 from builtins import range
30 from builtins import object
31 import sys
32 
33 import numpy
34 import warnings
35 from functools import reduce
36 
37 from lsst.afw.table import SourceCatalog
38 from lsst.log import Log
39 from lsst.pipe.base import Struct
40 from lsst.afw.cameraGeom import TAN_PIXELS
41 from lsst.afw.geom.ellipses import Quadrupole
42 import lsst.afw.geom as afwGeom
43 import lsst.pex.config as pexConfig
44 import lsst.afw.display.ds9 as ds9
45 from .starSelector import BaseStarSelectorTask, starSelectorRegistry
46 
47 
48 class ObjectSizeStarSelectorConfig(BaseStarSelectorTask.ConfigClass):
49  fluxMin = pexConfig.Field(
50  doc="specify the minimum psfFlux for good Psf Candidates",
51  dtype=float,
52  default=12500.0,
53  check=lambda x: x >= 0.0,
54  )
55  fluxMax = pexConfig.Field(
56  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
57  dtype=float,
58  default=0.0,
59  check=lambda x: x >= 0.0,
60  )
61  widthMin = pexConfig.Field(
62  doc="minimum width to include in histogram",
63  dtype=float,
64  default=0.0,
65  check=lambda x: x >= 0.0,
66  )
67  widthMax = pexConfig.Field(
68  doc="maximum width to include in histogram",
69  dtype=float,
70  default=10.0,
71  check=lambda x: x >= 0.0,
72  )
73  sourceFluxField = pexConfig.Field(
74  doc="Name of field in Source to use for flux measurement",
75  dtype=str,
76  default="base_GaussianFlux_flux",
77  )
78  widthStdAllowed = pexConfig.Field(
79  doc="Standard deviation of width allowed to be interpreted as good stars",
80  dtype=float,
81  default=0.15,
82  check=lambda x: x >= 0.0,
83  )
84  nSigmaClip = pexConfig.Field(
85  doc="Keep objects within this many sigma of cluster 0's median",
86  dtype=float,
87  default=2.0,
88  check=lambda x: x >= 0.0,
89  )
90 
91  def validate(self):
92  BaseStarSelectorTask.ConfigClass.validate(self)
93  if self.widthMin > self.widthMax:
94  raise pexConfig.FieldValidationError("widthMin (%f) > widthMax (%f)"
95  % (self.widthMin, self.widthMax))
96 
97 
98 class EventHandler(object):
99  """A class to handle key strokes with matplotlib displays"""
100 
101  def __init__(self, axes, xs, ys, x, y, frames=[0]):
102  self.axes = axes
103  self.xs = xs
104  self.ys = ys
105  self.x = x
106  self.y = y
107  self.frames = frames
108 
109  self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self)
110 
111  def __call__(self, ev):
112  if ev.inaxes != self.axes:
113  return
114 
115  if ev.key and ev.key in ("p"):
116  dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata)
117  dist[numpy.where(numpy.isnan(dist))] = 1e30
118 
119  which = numpy.where(dist == min(dist))
120 
121  x = self.x[which][0]
122  y = self.y[which][0]
123  for frame in self.frames:
124  ds9.pan(x, y, frame=frame)
125  ds9.cmdBuffer.flush()
126  else:
127  pass
128 
129 
130 def _assignClusters(yvec, centers):
131  """Return a vector of centerIds based on their distance to the centers"""
132  assert len(centers) > 0
133 
134  minDist = numpy.nan*numpy.ones_like(yvec)
135  clusterId = numpy.empty_like(yvec)
136  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
137  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
138  dbl.setLevel(dbl.INFO)
139 
140  # Make sure we are logging aall numpy warnings...
141  oldSettings = numpy.seterr(all="warn")
142  with warnings.catch_warnings(record=True) as w:
143  warnings.simplefilter("always")
144  for i, mean in enumerate(centers):
145  dist = abs(yvec - mean)
146  if i == 0:
147  update = dist == dist # True for all points
148  else:
149  update = dist < minDist
150  if w: # Only do if w is not empty i.e. contains a warning message
151  dbl.trace(str(w[-1]))
152 
153  minDist[update] = dist[update]
154  clusterId[update] = i
155  numpy.seterr(**oldSettings)
156 
157  return clusterId
158 
159 
160 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
161  """A classic k-means algorithm, clustering yvec into nCluster clusters
162 
163  Return the set of centres, and the cluster ID for each of the points
164 
165  If useMedian is true, use the median of the cluster as its centre, rather than
166  the traditional mean
167 
168  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
169  "e.g. why not use the Forgy or random partition initialization methods"
170  however, the approach adopted here seems to work well for the particular sorts of things
171  we're clustering in this application
172  """
173 
174  assert nCluster > 0
175 
176  mean0 = sorted(yvec)[len(yvec)//10] # guess
177  delta = mean0 * widthStdAllowed * 2.0
178  centers = mean0 + delta * numpy.arange(nCluster)
179 
180  func = numpy.median if useMedian else numpy.mean
181 
182  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
183  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
184  while True:
185  oclusterId = clusterId
186  clusterId = _assignClusters(yvec, centers)
187 
188  if numpy.all(clusterId == oclusterId):
189  break
190 
191  for i in range(nCluster):
192  # Only compute func if some points are available; otherwise, default to NaN.
193  pointsInCluster = (clusterId == i)
194  if numpy.any(pointsInCluster):
195  centers[i] = func(yvec[pointsInCluster])
196  else:
197  centers[i] = numpy.nan
198 
199  return centers, clusterId
200 
201 
202 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
203  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median"""
204 
205  nMember = sum(clusterId == clusterNum)
206  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
207  return clusterId
208  for iter in range(nIteration):
209  old_nMember = nMember
210 
211  inCluster0 = clusterId == clusterNum
212  yv = yvec[inCluster0]
213 
214  centers[clusterNum] = numpy.median(yv)
215  stdev = numpy.std(yv)
216 
217  syv = sorted(yv)
218  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
219  median = syv[int(0.5*nMember)]
220 
221  sd = stdev if stdev < stdev_iqr else stdev_iqr
222 
223  if False:
224  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
225  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
226  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
227  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
228 
229  nMember = sum(clusterId == clusterNum)
230  # 'sd < widthStdAllowed * median' prevents too much rejections
231  if nMember == old_nMember or sd < widthStdAllowed * median:
232  break
233 
234  return clusterId
235 
236 
237 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
238  magType="model", clear=True):
239 
240  log = Log.getLogger("objectSizeStarSelector.plot")
241  try:
242  import matplotlib.pyplot as plt
243  except ImportError as e:
244  log.warn("Unable to import matplotlib: %s", e)
245  return
246 
247  global fig
248  if not fig:
249  fig = plt.figure()
250  else:
251  if clear:
252  fig.clf()
253 
254  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
255 
256  xmin = sorted(mag)[int(0.05*len(mag))]
257  xmax = sorted(mag)[int(0.95*len(mag))]
258 
259  axes.set_xlim(-17.5, -13)
260  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
261  axes.set_ylim(0, 10)
262 
263  colors = ["r", "g", "b", "c", "m", "k", ]
264  for k, mean in enumerate(centers):
265  if k == 0:
266  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
267 
268  l = (clusterId == k)
269  axes.plot(mag[l], width[l], marker, markersize=markersize, markeredgewidth=markeredgewidth,
270  color=colors[k % len(colors)])
271 
272  l = (clusterId == -1)
273  axes.plot(mag[l], width[l], marker, markersize=markersize, markeredgewidth=markeredgewidth,
274  color='k')
275 
276  if clear:
277  axes.set_xlabel("Instrumental %s mag" % magType)
278  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
279 
280  return fig
281 
282 ## \addtogroup LSST_task_documentation
283 ## \{
284 ## \page ObjectSizeStarSelectorTask
285 ## \ref ObjectSizeStarSelectorTask_ "ObjectSizeStarSelectorTask"
286 ## \copybrief ObjectSizeStarSelectorTask
287 ## \}
288 
289 
290 class ObjectSizeStarSelectorTask(BaseStarSelectorTask):
291  """!A star selector that looks for a cluster of small objects in a size-magnitude plot
292 
293  @anchor ObjectSizeStarSelectorTask_
294 
295  @section meas_algorithms_objectSizeStarSelector_Contents Contents
296 
297  - @ref meas_algorithms_objectSizeStarSelector_Purpose
298  - @ref meas_algorithms_objectSizeStarSelector_Initialize
299  - @ref meas_algorithms_objectSizeStarSelector_IO
300  - @ref meas_algorithms_objectSizeStarSelector_Config
301  - @ref meas_algorithms_objectSizeStarSelector_Debug
302 
303  @section meas_algorithms_objectSizeStarSelector_Purpose Description
304 
305  A star selector that looks for a cluster of small objects in a size-magnitude plot.
306 
307  @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation
308 
309  @copydoc \_\_init\_\_
310 
311  @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task
312 
313  Like all star selectors, the main method is `run`.
314 
315  @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters
316 
317  See @ref ObjectSizeStarSelectorConfig
318 
319  @section meas_algorithms_objectSizeStarSelector_Debug Debug variables
320 
321  ObjectSizeStarSelectorTask has a debug dictionary with the following keys:
322  <dl>
323  <dt>display
324  <dd>bool; if True display debug information
325  <dt>displayExposure
326  <dd>bool; if True display the exposure and spatial cells
327  <dt>plotMagSize
328  <dd>bool: if True display the magnitude-size relation using matplotlib
329  <dt>dumpData
330  <dd>bool; if True dump data to a pickle file
331  </dl>
332 
333  For example, put something like:
334  @code{.py}
335  import lsstDebug
336  def DebugInfo(name):
337  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
338  if name.endswith("objectSizeStarSelector"):
339  di.display = True
340  di.displayExposure = True
341  di.plotMagSize = True
342 
343  return di
344 
345  lsstDebug.Info = DebugInfo
346  @endcode
347  into your `debug.py` file and run your task with the `--debug` flag.
348  """
349  ConfigClass = ObjectSizeStarSelectorConfig
350  usesMatches = False # selectStars does not use its matches argument
351 
352  def selectStars(self, exposure, sourceCat, matches=None):
353  """!Return a list of PSF candidates that represent likely stars
354 
355  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
356 
357  \param[in] exposure the exposure containing the sources
358  \param[in] sourceCat catalog of sources that may be stars (an lsst.afw.table.SourceCatalog)
359  \param[in] matches astrometric matches; ignored by this star selector
360 
361  \return an lsst.pipe.base.Struct containing:
362  - starCat catalog of selected stars (a subset of sourceCat)
363  """
364  import lsstDebug
365  display = lsstDebug.Info(__name__).display
366  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
367  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
368  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
369 
370  detector = exposure.getDetector()
371  pixToTanXYTransform = None
372  if detector is not None:
373  tanSys = detector.makeCameraSys(TAN_PIXELS)
374  pixToTanXYTransform = detector.getTransformMap().get(tanSys)
375  #
376  # Look at the distribution of stars in the magnitude-size plane
377  #
378  flux = sourceCat.get(self.config.sourceFluxField)
379 
380  xx = numpy.empty(len(sourceCat))
381  xy = numpy.empty_like(xx)
382  yy = numpy.empty_like(xx)
383  for i, source in enumerate(sourceCat):
384  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
385  if pixToTanXYTransform:
386  p = afwGeom.Point2D(source.getX(), source.getY())
387  linTransform = pixToTanXYTransform.linearizeForwardTransform(p).getLinear()
388  m = Quadrupole(Ixx, Iyy, Ixy)
389  m.transform(linTransform)
390  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
391 
392  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
393 
394  width = numpy.sqrt(0.5*(xx + yy))
395 
396  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
397  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
398  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
399  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
400  bad = numpy.logical_or(bad, width < self.config.widthMin)
401  bad = numpy.logical_or(bad, width > self.config.widthMax)
402  if self.config.fluxMax > 0:
403  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
404  good = numpy.logical_not(bad)
405 
406  if not numpy.any(good):
407  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
408 
409  mag = -2.5*numpy.log10(flux[good])
410  width = width[good]
411  #
412  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
413  # the initial peak (of, we presume, stars) from the galaxies
414  #
415  if dumpData:
416  import os
417  import pickle as pickle
418  _ii = 0
419  while True:
420  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
421  if not os.path.exists(pickleFile):
422  break
423  _ii += 1
424 
425  with open(pickleFile, "wb") as fd:
426  pickle.dump(mag, fd, -1)
427  pickle.dump(width, fd, -1)
428 
429  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
430  widthStdAllowed=self.config.widthStdAllowed)
431 
432  if display and plotMagSize:
433  fig = plot(mag, width, centers, clusterId,
434  magType=self.config.sourceFluxField.split(".")[-1].title(),
435  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
436  else:
437  fig = None
438 
439  clusterId = _improveCluster(width, centers, clusterId,
440  nsigma=self.config.nSigmaClip,
441  widthStdAllowed=self.config.widthStdAllowed)
442 
443  if display and plotMagSize:
444  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
445 
446  stellar = (clusterId == 0)
447  #
448  # We know enough to plot, if so requested
449  #
450  frame = 0
451 
452  if fig:
453  if display and displayExposure:
454  ds9.mtv(exposure.getMaskedImage(), frame=frame, title="PSF candidates")
455 
456  global eventHandler
457  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
458  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
459 
460  fig.show()
461 
462  while True:
463  try:
464  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
465  except EOFError:
466  reply = None
467  if not reply:
468  reply = "c"
469 
470  if reply:
471  if reply[0] == "h":
472  print("""\
473  We cluster the points; red are the stellar candidates and the other colours are other clusters.
474  Points labelled + are rejects from the cluster (only for cluster 0).
475 
476  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
477 
478  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
479  """)
480  elif reply[0] == "p":
481  import pdb
482  pdb.set_trace()
483  elif reply[0] == 'q':
484  sys.exit(1)
485  else:
486  break
487 
488  if display and displayExposure:
489  mi = exposure.getMaskedImage()
490 
491  with ds9.Buffering():
492  for i, source in enumerate(sourceCat):
493  if good[i]:
494  ctype = ds9.GREEN # star candidate
495  else:
496  ctype = ds9.RED # not star
497 
498  ds9.dot("+", source.getX() - mi.getX0(),
499  source.getY() - mi.getY0(), frame=frame, ctype=ctype)
500 
501  starCat = SourceCatalog(sourceCat.table)
502  goodSources = [s for g, s in zip(good, sourceCat) if g]
503  for isStellar, source in zip(stellar, goodSources):
504  if isStellar:
505  starCat.append(source)
506 
507  return Struct(
508  starCat=starCat,
509  )
510 
511 
512 starSelectorRegistry.register("objectSize", ObjectSizeStarSelectorTask)
A star selector that looks for a cluster of small objects in a size-magnitude plot.
def selectStars
Return a list of PSF candidates that represent likely stars.