lsst.meas.algorithms  15.0-11-g7db6e543+4
objectSizeStarSelector.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 import sys
24 
25 import numpy
26 import warnings
27 from functools import reduce
28 
29 from lsst.log import Log
30 from lsst.pipe.base import Struct
31 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
32 import lsst.afw.geom as afwGeom
33 import lsst.pex.config as pexConfig
34 import lsst.afw.display.ds9 as ds9
35 from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry
36 
37 
38 class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass):
39  fluxMin = pexConfig.Field(
40  doc="specify the minimum psfFlux for good Psf Candidates",
41  dtype=float,
42  default=12500.0,
43  check=lambda x: x >= 0.0,
44  )
45  fluxMax = pexConfig.Field(
46  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
47  dtype=float,
48  default=0.0,
49  check=lambda x: x >= 0.0,
50  )
51  widthMin = pexConfig.Field(
52  doc="minimum width to include in histogram",
53  dtype=float,
54  default=0.0,
55  check=lambda x: x >= 0.0,
56  )
57  widthMax = pexConfig.Field(
58  doc="maximum width to include in histogram",
59  dtype=float,
60  default=10.0,
61  check=lambda x: x >= 0.0,
62  )
63  sourceFluxField = pexConfig.Field(
64  doc="Name of field in Source to use for flux measurement",
65  dtype=str,
66  default="base_GaussianFlux_flux",
67  )
68  widthStdAllowed = pexConfig.Field(
69  doc="Standard deviation of width allowed to be interpreted as good stars",
70  dtype=float,
71  default=0.15,
72  check=lambda x: x >= 0.0,
73  )
74  nSigmaClip = pexConfig.Field(
75  doc="Keep objects within this many sigma of cluster 0's median",
76  dtype=float,
77  default=2.0,
78  check=lambda x: x >= 0.0,
79  )
80  badFlags = pexConfig.ListField(
81  doc="List of flags which cause a source to be rejected as bad",
82  dtype=str,
83  default=[
84  "base_PixelFlags_flag_edge",
85  "base_PixelFlags_flag_interpolatedCenter",
86  "base_PixelFlags_flag_saturatedCenter",
87  "base_PixelFlags_flag_crCenter",
88  "base_PixelFlags_flag_bad",
89  "base_PixelFlags_flag_interpolated",
90  ],
91  )
92 
93  def validate(self):
94  BaseSourceSelectorTask.ConfigClass.validate(self)
95  if self.widthMin > self.widthMax:
96  raise pexConfig.FieldValidationError("widthMin (%f) > widthMax (%f)"
97  % (self.widthMin, self.widthMax))
98 
99 
101  """A class to handle key strokes with matplotlib displays"""
102 
103  def __init__(self, axes, xs, ys, x, y, frames=[0]):
104  self.axes = axes
105  self.xs = xs
106  self.ys = ys
107  self.x = x
108  self.y = y
109  self.frames = frames
110 
111  self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self)
112 
113  def __call__(self, ev):
114  if ev.inaxes != self.axes:
115  return
116 
117  if ev.key and ev.key in ("p"):
118  dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata)
119  dist[numpy.where(numpy.isnan(dist))] = 1e30
120 
121  which = numpy.where(dist == min(dist))
122 
123  x = self.x[which][0]
124  y = self.y[which][0]
125  for frame in self.frames:
126  ds9.pan(x, y, frame=frame)
127  ds9.cmdBuffer.flush()
128  else:
129  pass
130 
131 
132 def _assignClusters(yvec, centers):
133  """Return a vector of centerIds based on their distance to the centers"""
134  assert len(centers) > 0
135 
136  minDist = numpy.nan*numpy.ones_like(yvec)
137  clusterId = numpy.empty_like(yvec)
138  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
139  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
140  dbl.setLevel(dbl.INFO)
141 
142  # Make sure we are logging aall numpy warnings...
143  oldSettings = numpy.seterr(all="warn")
144  with warnings.catch_warnings(record=True) as w:
145  warnings.simplefilter("always")
146  for i, mean in enumerate(centers):
147  dist = abs(yvec - mean)
148  if i == 0:
149  update = dist == dist # True for all points
150  else:
151  update = dist < minDist
152  if w: # Only do if w is not empty i.e. contains a warning message
153  dbl.trace(str(w[-1]))
154 
155  minDist[update] = dist[update]
156  clusterId[update] = i
157  numpy.seterr(**oldSettings)
158 
159  return clusterId
160 
161 
162 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
163  """A classic k-means algorithm, clustering yvec into nCluster clusters
164 
165  Return the set of centres, and the cluster ID for each of the points
166 
167  If useMedian is true, use the median of the cluster as its centre, rather than
168  the traditional mean
169 
170  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
171  "e.g. why not use the Forgy or random partition initialization methods"
172  however, the approach adopted here seems to work well for the particular sorts of things
173  we're clustering in this application
174  """
175 
176  assert nCluster > 0
177 
178  mean0 = sorted(yvec)[len(yvec)//10] # guess
179  delta = mean0 * widthStdAllowed * 2.0
180  centers = mean0 + delta * numpy.arange(nCluster)
181 
182  func = numpy.median if useMedian else numpy.mean
183 
184  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
185  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
186  while True:
187  oclusterId = clusterId
188  clusterId = _assignClusters(yvec, centers)
189 
190  if numpy.all(clusterId == oclusterId):
191  break
192 
193  for i in range(nCluster):
194  # Only compute func if some points are available; otherwise, default to NaN.
195  pointsInCluster = (clusterId == i)
196  if numpy.any(pointsInCluster):
197  centers[i] = func(yvec[pointsInCluster])
198  else:
199  centers[i] = numpy.nan
200 
201  return centers, clusterId
202 
203 
204 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
205  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median"""
206 
207  nMember = sum(clusterId == clusterNum)
208  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
209  return clusterId
210  for iter in range(nIteration):
211  old_nMember = nMember
212 
213  inCluster0 = clusterId == clusterNum
214  yv = yvec[inCluster0]
215 
216  centers[clusterNum] = numpy.median(yv)
217  stdev = numpy.std(yv)
218 
219  syv = sorted(yv)
220  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
221  median = syv[int(0.5*nMember)]
222 
223  sd = stdev if stdev < stdev_iqr else stdev_iqr
224 
225  if False:
226  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
227  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
228  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
229  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
230 
231  nMember = sum(clusterId == clusterNum)
232  # 'sd < widthStdAllowed * median' prevents too much rejections
233  if nMember == old_nMember or sd < widthStdAllowed * median:
234  break
235 
236  return clusterId
237 
238 
239 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
240  magType="model", clear=True):
241 
242  log = Log.getLogger("objectSizeStarSelector.plot")
243  try:
244  import matplotlib.pyplot as plt
245  except ImportError as e:
246  log.warn("Unable to import matplotlib: %s", e)
247  return
248 
249  global fig
250  if not fig:
251  fig = plt.figure()
252  else:
253  if clear:
254  fig.clf()
255 
256  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
257 
258  xmin = sorted(mag)[int(0.05*len(mag))]
259  xmax = sorted(mag)[int(0.95*len(mag))]
260 
261  axes.set_xlim(-17.5, -13)
262  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
263  axes.set_ylim(0, 10)
264 
265  colors = ["r", "g", "b", "c", "m", "k", ]
266  for k, mean in enumerate(centers):
267  if k == 0:
268  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
269 
270  li = (clusterId == k)
271  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
272  color=colors[k % len(colors)])
273 
274  li = (clusterId == -1)
275  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
276  color='k')
277 
278  if clear:
279  axes.set_xlabel("Instrumental %s mag" % magType)
280  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
281 
282  return fig
283 
284 
290 
291 
292 @pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry)
294  """!A star selector that looks for a cluster of small objects in a size-magnitude plot
295 
296  @anchor ObjectSizeStarSelectorTask_
297 
298  @section meas_algorithms_objectSizeStarSelector_Contents Contents
299 
300  - @ref meas_algorithms_objectSizeStarSelector_Purpose
301  - @ref meas_algorithms_objectSizeStarSelector_Initialize
302  - @ref meas_algorithms_objectSizeStarSelector_IO
303  - @ref meas_algorithms_objectSizeStarSelector_Config
304  - @ref meas_algorithms_objectSizeStarSelector_Debug
305 
306  @section meas_algorithms_objectSizeStarSelector_Purpose Description
307 
308  A star selector that looks for a cluster of small objects in a size-magnitude plot.
309 
310  @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation
311 
312  @copydoc \_\_init\_\_
313 
314  @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task
315 
316  Like all star selectors, the main method is `run`.
317 
318  @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters
319 
320  See @ref ObjectSizeStarSelectorConfig
321 
322  @section meas_algorithms_objectSizeStarSelector_Debug Debug variables
323 
324  ObjectSizeStarSelectorTask has a debug dictionary with the following keys:
325  <dl>
326  <dt>display
327  <dd>bool; if True display debug information
328  <dt>displayExposure
329  <dd>bool; if True display the exposure and spatial cells
330  <dt>plotMagSize
331  <dd>bool: if True display the magnitude-size relation using matplotlib
332  <dt>dumpData
333  <dd>bool; if True dump data to a pickle file
334  </dl>
335 
336  For example, put something like:
337  @code{.py}
338  import lsstDebug
339  def DebugInfo(name):
340  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
341  if name.endswith("objectSizeStarSelector"):
342  di.display = True
343  di.displayExposure = True
344  di.plotMagSize = True
345 
346  return di
347 
348  lsstDebug.Info = DebugInfo
349  @endcode
350  into your `debug.py` file and run your task with the `--debug` flag.
351  """
352  ConfigClass = ObjectSizeStarSelectorConfig
353  usesMatches = False # selectStars does not use its matches argument
354 
355  def selectSources(self, sourceCat, matches=None, exposure=None):
356  """Return a selection of PSF candidates that represent likely stars.
357 
358  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
359 
360  Parameters:
361  -----------
362  sourceCat : `lsst.afw.table.SourceCatalog`
363  Catalog of sources to select from.
364  This catalog must be contiguous in memory.
365  matches : `list` of `lsst.afw.table.ReferenceMatch` or None
366  Ignored in this SourceSelector.
367  exposure : `lsst.afw.image.Exposure` or None
368  The exposure the catalog was built from; used to get the detector
369  to transform to TanPix, and for debug display.
370 
371  Return
372  ------
373  struct : `lsst.pipe.base.Struct`
374  The struct contains the following data:
375 
376  - selected : `array` of `bool``
377  Boolean array of sources that were selected, same length as
378  sourceCat.
379  """
380  import lsstDebug
381  display = lsstDebug.Info(__name__).display
382  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
383  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
384  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
385 
386  detector = exposure.getDetector()
387  pixToTanPix = None
388  if detector is not None:
389  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
390  #
391  # Look at the distribution of stars in the magnitude-size plane
392  #
393  flux = sourceCat.get(self.config.sourceFluxField)
394 
395  xx = numpy.empty(len(sourceCat))
396  xy = numpy.empty_like(xx)
397  yy = numpy.empty_like(xx)
398  for i, source in enumerate(sourceCat):
399  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
400  if pixToTanPix:
401  p = afwGeom.Point2D(source.getX(), source.getY())
402  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
403  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
404  m.transform(linTransform)
405  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
406 
407  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
408 
409  width = numpy.sqrt(0.5*(xx + yy))
410  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
411  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
412  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
413  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
414  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
415  bad = numpy.logical_or(bad, width < self.config.widthMin)
416  bad = numpy.logical_or(bad, width > self.config.widthMax)
417  if self.config.fluxMax > 0:
418  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
419  good = numpy.logical_not(bad)
420 
421  if not numpy.any(good):
422  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
423 
424  mag = -2.5*numpy.log10(flux[good])
425  width = width[good]
426  #
427  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
428  # the initial peak (of, we presume, stars) from the galaxies
429  #
430  if dumpData:
431  import os
432  import pickle as pickle
433  _ii = 0
434  while True:
435  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
436  if not os.path.exists(pickleFile):
437  break
438  _ii += 1
439 
440  with open(pickleFile, "wb") as fd:
441  pickle.dump(mag, fd, -1)
442  pickle.dump(width, fd, -1)
443 
444  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
445  widthStdAllowed=self.config.widthStdAllowed)
446 
447  if display and plotMagSize:
448  fig = plot(mag, width, centers, clusterId,
449  magType=self.config.sourceFluxField.split(".")[-1].title(),
450  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
451  else:
452  fig = None
453 
454  clusterId = _improveCluster(width, centers, clusterId,
455  nsigma=self.config.nSigmaClip,
456  widthStdAllowed=self.config.widthStdAllowed)
457 
458  if display and plotMagSize:
459  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
460 
461  stellar = (clusterId == 0)
462  #
463  # We know enough to plot, if so requested
464  #
465  frame = 0
466 
467  if fig:
468  if display and displayExposure:
469  ds9.mtv(exposure.getMaskedImage(), frame=frame, title="PSF candidates")
470 
471  global eventHandler
472  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
473  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
474 
475  fig.show()
476 
477  while True:
478  try:
479  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
480  except EOFError:
481  reply = None
482  if not reply:
483  reply = "c"
484 
485  if reply:
486  if reply[0] == "h":
487  print("""\
488  We cluster the points; red are the stellar candidates and the other colours are other clusters.
489  Points labelled + are rejects from the cluster (only for cluster 0).
490 
491  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
492 
493  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
494  """)
495  elif reply[0] == "p":
496  import pdb
497  pdb.set_trace()
498  elif reply[0] == 'q':
499  sys.exit(1)
500  else:
501  break
502 
503  if display and displayExposure:
504  mi = exposure.getMaskedImage()
505 
506  with ds9.Buffering():
507  for i, source in enumerate(sourceCat):
508  if good[i]:
509  ctype = ds9.GREEN # star candidate
510  else:
511  ctype = ds9.RED # not star
512 
513  ds9.dot("+", source.getX() - mi.getX0(),
514  source.getY() - mi.getY0(), frame=frame, ctype=ctype)
515 
516  # stellar only applies to good==True objects
517  mask = good == True # noqa (numpy bool comparison): E712
518  good[mask] = stellar
519 
520  return Struct(selected=good)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)
A star selector that looks for a cluster of small objects in a size-magnitude plot.
AffineTransform linearizeTransform(TransformPoint2ToPoint2 const &original, Point2D const &inPoint)