lsst.meas.algorithms  16.0-29-g31fb3001
objectSizeStarSelector.py
Go to the documentation of this file.
1 # This file is part of meas_algorithms.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import sys
23 
24 import numpy
25 import warnings
26 from functools import reduce
27 
28 from lsst.log import Log
29 from lsst.pipe.base import Struct
30 import lsst.geom
31 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
32 import lsst.afw.geom as afwGeom
33 import lsst.pex.config as pexConfig
34 import lsst.afw.display.ds9 as ds9
35 from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry
36 
37 
38 class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass):
39  doFluxLimit = pexConfig.Field(
40  doc="Apply flux limit to Psf Candidate selection?",
41  dtype=bool,
42  default=True,
43  )
44  fluxMin = pexConfig.Field(
45  doc="specify the minimum psfFlux for good Psf Candidates",
46  dtype=float,
47  default=12500.0,
48  check=lambda x: x >= 0.0,
49  )
50  fluxMax = pexConfig.Field(
51  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
52  dtype=float,
53  default=0.0,
54  check=lambda x: x >= 0.0,
55  )
56  doSignalToNoiseLimit = pexConfig.Field(
57  doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?",
58  dtype=bool,
59  default=False,
60  )
61  signalToNoiseMin = pexConfig.Field(
62  doc="specify the minimum signal-to-noise for good Psf Candidates",
63  dtype=float,
64  default=20.0,
65  check=lambda x: x >= 0.0,
66  )
67  signalToNoiseMax = pexConfig.Field(
68  doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)",
69  dtype=float,
70  default=0.0,
71  check=lambda x: x >= 0.0,
72  )
73  widthMin = pexConfig.Field(
74  doc="minimum width to include in histogram",
75  dtype=float,
76  default=0.0,
77  check=lambda x: x >= 0.0,
78  )
79  widthMax = pexConfig.Field(
80  doc="maximum width to include in histogram",
81  dtype=float,
82  default=10.0,
83  check=lambda x: x >= 0.0,
84  )
85  sourceFluxField = pexConfig.Field(
86  doc="Name of field in Source to use for flux measurement",
87  dtype=str,
88  default="base_GaussianFlux_instFlux",
89  )
90  widthStdAllowed = pexConfig.Field(
91  doc="Standard deviation of width allowed to be interpreted as good stars",
92  dtype=float,
93  default=0.15,
94  check=lambda x: x >= 0.0,
95  )
96  nSigmaClip = pexConfig.Field(
97  doc="Keep objects within this many sigma of cluster 0's median",
98  dtype=float,
99  default=2.0,
100  check=lambda x: x >= 0.0,
101  )
102  badFlags = pexConfig.ListField(
103  doc="List of flags which cause a source to be rejected as bad",
104  dtype=str,
105  default=[
106  "base_PixelFlags_flag_edge",
107  "base_PixelFlags_flag_interpolatedCenter",
108  "base_PixelFlags_flag_saturatedCenter",
109  "base_PixelFlags_flag_crCenter",
110  "base_PixelFlags_flag_bad",
111  "base_PixelFlags_flag_interpolated",
112  ],
113  )
114 
115  def validate(self):
116  BaseSourceSelectorTask.ConfigClass.validate(self)
117  if self.widthMin > self.widthMax:
118  raise pexConfig.FieldValidationError("widthMin (%f) > widthMax (%f)"
119  % (self.widthMin, self.widthMax))
120 
121 
123  """A class to handle key strokes with matplotlib displays"""
124 
125  def __init__(self, axes, xs, ys, x, y, frames=[0]):
126  self.axes = axes
127  self.xs = xs
128  self.ys = ys
129  self.x = x
130  self.y = y
131  self.frames = frames
132 
133  self.cid = self.axes.figure.canvas.mpl_connect('key_press_event', self)
134 
135  def __call__(self, ev):
136  if ev.inaxes != self.axes:
137  return
138 
139  if ev.key and ev.key in ("p"):
140  dist = numpy.hypot(self.xs - ev.xdata, self.ys - ev.ydata)
141  dist[numpy.where(numpy.isnan(dist))] = 1e30
142 
143  which = numpy.where(dist == min(dist))
144 
145  x = self.x[which][0]
146  y = self.y[which][0]
147  for frame in self.frames:
148  ds9.pan(x, y, frame=frame)
149  ds9.cmdBuffer.flush()
150  else:
151  pass
152 
153 
154 def _assignClusters(yvec, centers):
155  """Return a vector of centerIds based on their distance to the centers"""
156  assert len(centers) > 0
157 
158  minDist = numpy.nan*numpy.ones_like(yvec)
159  clusterId = numpy.empty_like(yvec)
160  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
161  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
162  dbl.setLevel(dbl.INFO)
163 
164  # Make sure we are logging aall numpy warnings...
165  oldSettings = numpy.seterr(all="warn")
166  with warnings.catch_warnings(record=True) as w:
167  warnings.simplefilter("always")
168  for i, mean in enumerate(centers):
169  dist = abs(yvec - mean)
170  if i == 0:
171  update = dist == dist # True for all points
172  else:
173  update = dist < minDist
174  if w: # Only do if w is not empty i.e. contains a warning message
175  dbl.trace(str(w[-1]))
176 
177  minDist[update] = dist[update]
178  clusterId[update] = i
179  numpy.seterr(**oldSettings)
180 
181  return clusterId
182 
183 
184 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
185  """A classic k-means algorithm, clustering yvec into nCluster clusters
186 
187  Return the set of centres, and the cluster ID for each of the points
188 
189  If useMedian is true, use the median of the cluster as its centre, rather than
190  the traditional mean
191 
192  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
193  "e.g. why not use the Forgy or random partition initialization methods"
194  however, the approach adopted here seems to work well for the particular sorts of things
195  we're clustering in this application
196  """
197 
198  assert nCluster > 0
199 
200  mean0 = sorted(yvec)[len(yvec)//10] # guess
201  delta = mean0 * widthStdAllowed * 2.0
202  centers = mean0 + delta * numpy.arange(nCluster)
203 
204  func = numpy.median if useMedian else numpy.mean
205 
206  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
207  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
208  while True:
209  oclusterId = clusterId
210  clusterId = _assignClusters(yvec, centers)
211 
212  if numpy.all(clusterId == oclusterId):
213  break
214 
215  for i in range(nCluster):
216  # Only compute func if some points are available; otherwise, default to NaN.
217  pointsInCluster = (clusterId == i)
218  if numpy.any(pointsInCluster):
219  centers[i] = func(yvec[pointsInCluster])
220  else:
221  centers[i] = numpy.nan
222 
223  return centers, clusterId
224 
225 
226 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
227  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median"""
228 
229  nMember = sum(clusterId == clusterNum)
230  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
231  return clusterId
232  for iter in range(nIteration):
233  old_nMember = nMember
234 
235  inCluster0 = clusterId == clusterNum
236  yv = yvec[inCluster0]
237 
238  centers[clusterNum] = numpy.median(yv)
239  stdev = numpy.std(yv)
240 
241  syv = sorted(yv)
242  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
243  median = syv[int(0.5*nMember)]
244 
245  sd = stdev if stdev < stdev_iqr else stdev_iqr
246 
247  if False:
248  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
249  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
250  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
251  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
252 
253  nMember = sum(clusterId == clusterNum)
254  # 'sd < widthStdAllowed * median' prevents too much rejections
255  if nMember == old_nMember or sd < widthStdAllowed * median:
256  break
257 
258  return clusterId
259 
260 
261 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
262  magType="model", clear=True):
263 
264  log = Log.getLogger("objectSizeStarSelector.plot")
265  try:
266  import matplotlib.pyplot as plt
267  except ImportError as e:
268  log.warn("Unable to import matplotlib: %s", e)
269  return
270 
271  global fig
272  if not fig:
273  fig = plt.figure()
274  else:
275  if clear:
276  fig.clf()
277 
278  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
279 
280  xmin = sorted(mag)[int(0.05*len(mag))]
281  xmax = sorted(mag)[int(0.95*len(mag))]
282 
283  axes.set_xlim(-17.5, -13)
284  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
285  axes.set_ylim(0, 10)
286 
287  colors = ["r", "g", "b", "c", "m", "k", ]
288  for k, mean in enumerate(centers):
289  if k == 0:
290  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
291 
292  li = (clusterId == k)
293  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
294  color=colors[k % len(colors)])
295 
296  li = (clusterId == -1)
297  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
298  color='k')
299 
300  if clear:
301  axes.set_xlabel("Instrumental %s mag" % magType)
302  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
303 
304  return fig
305 
306 
312 
313 
314 @pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry)
316  r"""!A star selector that looks for a cluster of small objects in a size-magnitude plot
317 
318  @anchor ObjectSizeStarSelectorTask_
319 
320  @section meas_algorithms_objectSizeStarSelector_Contents Contents
321 
322  - @ref meas_algorithms_objectSizeStarSelector_Purpose
323  - @ref meas_algorithms_objectSizeStarSelector_Initialize
324  - @ref meas_algorithms_objectSizeStarSelector_IO
325  - @ref meas_algorithms_objectSizeStarSelector_Config
326  - @ref meas_algorithms_objectSizeStarSelector_Debug
327 
328  @section meas_algorithms_objectSizeStarSelector_Purpose Description
329 
330  A star selector that looks for a cluster of small objects in a size-magnitude plot.
331 
332  @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation
333 
334  @copydoc \_\_init\_\_
335 
336  @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task
337 
338  Like all star selectors, the main method is `run`.
339 
340  @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters
341 
342  See @ref ObjectSizeStarSelectorConfig
343 
344  @section meas_algorithms_objectSizeStarSelector_Debug Debug variables
345 
346  ObjectSizeStarSelectorTask has a debug dictionary with the following keys:
347  <dl>
348  <dt>display
349  <dd>bool; if True display debug information
350  <dt>displayExposure
351  <dd>bool; if True display the exposure and spatial cells
352  <dt>plotMagSize
353  <dd>bool: if True display the magnitude-size relation using matplotlib
354  <dt>dumpData
355  <dd>bool; if True dump data to a pickle file
356  </dl>
357 
358  For example, put something like:
359  @code{.py}
360  import lsstDebug
361  def DebugInfo(name):
362  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
363  if name.endswith("objectSizeStarSelector"):
364  di.display = True
365  di.displayExposure = True
366  di.plotMagSize = True
367 
368  return di
369 
370  lsstDebug.Info = DebugInfo
371  @endcode
372  into your `debug.py` file and run your task with the `--debug` flag.
373  """
374  ConfigClass = ObjectSizeStarSelectorConfig
375  usesMatches = False # selectStars does not use its matches argument
376 
377  def selectSources(self, sourceCat, matches=None, exposure=None):
378  """Return a selection of PSF candidates that represent likely stars.
379 
380  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
381 
382  Parameters:
383  -----------
384  sourceCat : `lsst.afw.table.SourceCatalog`
385  Catalog of sources to select from.
386  This catalog must be contiguous in memory.
387  matches : `list` of `lsst.afw.table.ReferenceMatch` or None
388  Ignored in this SourceSelector.
389  exposure : `lsst.afw.image.Exposure` or None
390  The exposure the catalog was built from; used to get the detector
391  to transform to TanPix, and for debug display.
392 
393  Return
394  ------
395  struct : `lsst.pipe.base.Struct`
396  The struct contains the following data:
397 
398  - selected : `array` of `bool``
399  Boolean array of sources that were selected, same length as
400  sourceCat.
401  """
402  import lsstDebug
403  display = lsstDebug.Info(__name__).display
404  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
405  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
406  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
407 
408  detector = None
409  pixToTanPix = None
410  if exposure:
411  detector = exposure.getDetector()
412  if detector:
413  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
414  #
415  # Look at the distribution of stars in the magnitude-size plane
416  #
417  flux = sourceCat.get(self.config.sourceFluxField)
418  fluxErr = sourceCat.get(self.config.sourceFluxField + "Err")
419 
420  xx = numpy.empty(len(sourceCat))
421  xy = numpy.empty_like(xx)
422  yy = numpy.empty_like(xx)
423  for i, source in enumerate(sourceCat):
424  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
425  if pixToTanPix:
426  p = lsst.geom.Point2D(source.getX(), source.getY())
427  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
428  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
429  m.transform(linTransform)
430  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
431 
432  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
433 
434  width = numpy.sqrt(0.5*(xx + yy))
435  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
436  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
437  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
438  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
439  if self.config.doFluxLimit:
440  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
441  if self.config.fluxMax > 0:
442  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
443  if self.config.doSignalToNoiseLimit:
444  bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin)
445  if self.config.signalToNoiseMax > 0:
446  bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax)
447  bad = numpy.logical_or(bad, width < self.config.widthMin)
448  bad = numpy.logical_or(bad, width > self.config.widthMax)
449  good = numpy.logical_not(bad)
450 
451  if not numpy.any(good):
452  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
453 
454  mag = -2.5*numpy.log10(flux[good])
455  width = width[good]
456  #
457  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
458  # the initial peak (of, we presume, stars) from the galaxies
459  #
460  if dumpData:
461  import os
462  import pickle as pickle
463  _ii = 0
464  while True:
465  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
466  if not os.path.exists(pickleFile):
467  break
468  _ii += 1
469 
470  with open(pickleFile, "wb") as fd:
471  pickle.dump(mag, fd, -1)
472  pickle.dump(width, fd, -1)
473 
474  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
475  widthStdAllowed=self.config.widthStdAllowed)
476 
477  if display and plotMagSize:
478  fig = plot(mag, width, centers, clusterId,
479  magType=self.config.sourceFluxField.split(".")[-1].title(),
480  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
481  else:
482  fig = None
483 
484  clusterId = _improveCluster(width, centers, clusterId,
485  nsigma=self.config.nSigmaClip,
486  widthStdAllowed=self.config.widthStdAllowed)
487 
488  if display and plotMagSize:
489  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
490 
491  stellar = (clusterId == 0)
492  #
493  # We know enough to plot, if so requested
494  #
495  frame = 0
496 
497  if fig:
498  if display and displayExposure:
499  ds9.mtv(exposure.getMaskedImage(), frame=frame, title="PSF candidates")
500 
501  global eventHandler
502  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
503  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
504 
505  fig.show()
506 
507  while True:
508  try:
509  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
510  except EOFError:
511  reply = None
512  if not reply:
513  reply = "c"
514 
515  if reply:
516  if reply[0] == "h":
517  print("""\
518  We cluster the points; red are the stellar candidates and the other colours are other clusters.
519  Points labelled + are rejects from the cluster (only for cluster 0).
520 
521  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
522 
523  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
524  """)
525  elif reply[0] == "p":
526  import pdb
527  pdb.set_trace()
528  elif reply[0] == 'q':
529  sys.exit(1)
530  else:
531  break
532 
533  if display and displayExposure:
534  mi = exposure.getMaskedImage()
535 
536  with ds9.Buffering():
537  for i, source in enumerate(sourceCat):
538  if good[i]:
539  ctype = ds9.GREEN # star candidate
540  else:
541  ctype = ds9.RED # not star
542 
543  ds9.dot("+", source.getX() - mi.getX0(),
544  source.getY() - mi.getY0(), frame=frame, ctype=ctype)
545 
546  # stellar only applies to good==True objects
547  mask = good == True # noqa (numpy bool comparison): E712
548  good[mask] = stellar
549 
550  return Struct(selected=good)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)
A star selector that looks for a cluster of small objects in a size-magnitude plot.