lsst.meas.algorithms  21.0.0-20-g8cd22d88+2f055bbffc
objectSizeStarSelector.py
Go to the documentation of this file.
1 # This file is part of meas_algorithms.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import sys
23 
24 import numpy
25 import warnings
26 from functools import reduce
27 
28 from lsst.log import Log
29 from lsst.pipe.base import Struct
30 import lsst.geom
31 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
32 import lsst.afw.geom as afwGeom
33 import lsst.pex.config as pexConfig
34 import lsst.afw.display as afwDisplay
35 from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry
36 
37 afwDisplay.setDefaultMaskTransparency(75)
38 
39 
40 class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass):
41  doFluxLimit = pexConfig.Field(
42  doc="Apply flux limit to Psf Candidate selection?",
43  dtype=bool,
44  default=True,
45  )
46  fluxMin = pexConfig.Field(
47  doc="specify the minimum psfFlux for good Psf Candidates",
48  dtype=float,
49  default=12500.0,
50  check=lambda x: x >= 0.0,
51  )
52  fluxMax = pexConfig.Field(
53  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
54  dtype=float,
55  default=0.0,
56  check=lambda x: x >= 0.0,
57  )
58  doSignalToNoiseLimit = pexConfig.Field(
59  doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?",
60  dtype=bool,
61  default=False,
62  )
63  signalToNoiseMin = pexConfig.Field(
64  doc="specify the minimum signal-to-noise for good Psf Candidates",
65  dtype=float,
66  default=20.0,
67  check=lambda x: x >= 0.0,
68  )
69  signalToNoiseMax = pexConfig.Field(
70  doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)",
71  dtype=float,
72  default=0.0,
73  check=lambda x: x >= 0.0,
74  )
75  widthMin = pexConfig.Field(
76  doc="minimum width to include in histogram",
77  dtype=float,
78  default=0.0,
79  check=lambda x: x >= 0.0,
80  )
81  widthMax = pexConfig.Field(
82  doc="maximum width to include in histogram",
83  dtype=float,
84  default=10.0,
85  check=lambda x: x >= 0.0,
86  )
87  sourceFluxField = pexConfig.Field(
88  doc="Name of field in Source to use for flux measurement",
89  dtype=str,
90  default="base_GaussianFlux_instFlux",
91  )
92  widthStdAllowed = pexConfig.Field(
93  doc="Standard deviation of width allowed to be interpreted as good stars",
94  dtype=float,
95  default=0.15,
96  check=lambda x: x >= 0.0,
97  )
98  nSigmaClip = pexConfig.Field(
99  doc="Keep objects within this many sigma of cluster 0's median",
100  dtype=float,
101  default=2.0,
102  check=lambda x: x >= 0.0,
103  )
104  badFlags = pexConfig.ListField(
105  doc="List of flags which cause a source to be rejected as bad",
106  dtype=str,
107  default=[
108  "base_PixelFlags_flag_edge",
109  "base_PixelFlags_flag_interpolatedCenter",
110  "base_PixelFlags_flag_saturatedCenter",
111  "base_PixelFlags_flag_crCenter",
112  "base_PixelFlags_flag_bad",
113  "base_PixelFlags_flag_interpolated",
114  ],
115  )
116 
117  def validate(self):
118  BaseSourceSelectorTask.ConfigClass.validate(self)
119  if self.widthMinwidthMin > self.widthMaxwidthMax:
120  msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})"
121  raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg)
122 
123 
125  """A class to handle key strokes with matplotlib displays"""
126 
127  def __init__(self, axes, xs, ys, x, y, frames=[0]):
128  self.axesaxes = axes
129  self.xsxs = xs
130  self.ysys = ys
131  self.xx = x
132  self.yy = y
133  self.framesframes = frames
134 
135  self.cidcid = self.axesaxes.figure.canvas.mpl_connect('key_press_event', self)
136 
137  def __call__(self, ev):
138  if ev.inaxes != self.axesaxes:
139  return
140 
141  if ev.key and ev.key in ("p"):
142  dist = numpy.hypot(self.xsxs - ev.xdata, self.ysys - ev.ydata)
143  dist[numpy.where(numpy.isnan(dist))] = 1e30
144 
145  which = numpy.where(dist == min(dist))
146 
147  x = self.xx[which][0]
148  y = self.yy[which][0]
149  for frame in self.framesframes:
150  disp = afwDisplay.Display(frame=frame)
151  disp.pan(x, y)
152  disp.flush()
153  else:
154  pass
155 
156 
157 def _assignClusters(yvec, centers):
158  """Return a vector of centerIds based on their distance to the centers"""
159  assert len(centers) > 0
160 
161  minDist = numpy.nan*numpy.ones_like(yvec)
162  clusterId = numpy.empty_like(yvec)
163  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
164  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
165  dbl.setLevel(dbl.INFO)
166 
167  # Make sure we are logging aall numpy warnings...
168  oldSettings = numpy.seterr(all="warn")
169  with warnings.catch_warnings(record=True) as w:
170  warnings.simplefilter("always")
171  for i, mean in enumerate(centers):
172  dist = abs(yvec - mean)
173  if i == 0:
174  update = dist == dist # True for all points
175  else:
176  update = dist < minDist
177  if w: # Only do if w is not empty i.e. contains a warning message
178  dbl.trace(str(w[-1]))
179 
180  minDist[update] = dist[update]
181  clusterId[update] = i
182  numpy.seterr(**oldSettings)
183 
184  return clusterId
185 
186 
187 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
188  """A classic k-means algorithm, clustering yvec into nCluster clusters
189 
190  Return the set of centres, and the cluster ID for each of the points
191 
192  If useMedian is true, use the median of the cluster as its centre, rather than
193  the traditional mean
194 
195  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
196  "e.g. why not use the Forgy or random partition initialization methods"
197  however, the approach adopted here seems to work well for the particular sorts of things
198  we're clustering in this application
199  """
200 
201  assert nCluster > 0
202 
203  mean0 = sorted(yvec)[len(yvec)//10] # guess
204  delta = mean0 * widthStdAllowed * 2.0
205  centers = mean0 + delta * numpy.arange(nCluster)
206 
207  func = numpy.median if useMedian else numpy.mean
208 
209  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
210  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
211  while True:
212  oclusterId = clusterId
213  clusterId = _assignClusters(yvec, centers)
214 
215  if numpy.all(clusterId == oclusterId):
216  break
217 
218  for i in range(nCluster):
219  # Only compute func if some points are available; otherwise, default to NaN.
220  pointsInCluster = (clusterId == i)
221  if numpy.any(pointsInCluster):
222  centers[i] = func(yvec[pointsInCluster])
223  else:
224  centers[i] = numpy.nan
225 
226  return centers, clusterId
227 
228 
229 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
230  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median"""
231 
232  nMember = sum(clusterId == clusterNum)
233  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
234  return clusterId
235  for iter in range(nIteration):
236  old_nMember = nMember
237 
238  inCluster0 = clusterId == clusterNum
239  yv = yvec[inCluster0]
240 
241  centers[clusterNum] = numpy.median(yv)
242  stdev = numpy.std(yv)
243 
244  syv = sorted(yv)
245  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
246  median = syv[int(0.5*nMember)]
247 
248  sd = stdev if stdev < stdev_iqr else stdev_iqr
249 
250  if False:
251  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
252  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
253  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
254  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
255 
256  nMember = sum(clusterId == clusterNum)
257  # 'sd < widthStdAllowed * median' prevents too much rejections
258  if nMember == old_nMember or sd < widthStdAllowed * median:
259  break
260 
261  return clusterId
262 
263 
264 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
265  magType="model", clear=True):
266 
267  log = Log.getLogger("objectSizeStarSelector.plot")
268  try:
269  import matplotlib.pyplot as plt
270  except ImportError as e:
271  log.warn("Unable to import matplotlib: %s", e)
272  return
273 
274  try:
275  fig
276  except NameError:
277  fig = plt.figure()
278  else:
279  if clear:
280  fig.clf()
281 
282  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
283 
284  xmin = sorted(mag)[int(0.05*len(mag))]
285  xmax = sorted(mag)[int(0.95*len(mag))]
286 
287  axes.set_xlim(-17.5, -13)
288  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
289  axes.set_ylim(0, 10)
290 
291  colors = ["r", "g", "b", "c", "m", "k", ]
292  for k, mean in enumerate(centers):
293  if k == 0:
294  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
295 
296  li = (clusterId == k)
297  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
298  color=colors[k % len(colors)])
299 
300  li = (clusterId == -1)
301  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
302  color='k')
303 
304  if clear:
305  axes.set_xlabel("Instrumental %s mag" % magType)
306  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
307 
308  return fig
309 
310 
316 
317 
318 @pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry)
320  r"""!A star selector that looks for a cluster of small objects in a size-magnitude plot
321 
322  @anchor ObjectSizeStarSelectorTask_
323 
324  @section meas_algorithms_objectSizeStarSelector_Contents Contents
325 
326  - @ref meas_algorithms_objectSizeStarSelector_Purpose
327  - @ref meas_algorithms_objectSizeStarSelector_Initialize
328  - @ref meas_algorithms_objectSizeStarSelector_IO
329  - @ref meas_algorithms_objectSizeStarSelector_Config
330  - @ref meas_algorithms_objectSizeStarSelector_Debug
331 
332  @section meas_algorithms_objectSizeStarSelector_Purpose Description
333 
334  A star selector that looks for a cluster of small objects in a size-magnitude plot.
335 
336  @section meas_algorithms_objectSizeStarSelector_Initialize Task initialisation
337 
338  @copydoc \_\_init\_\_
339 
340  @section meas_algorithms_objectSizeStarSelector_IO Invoking the Task
341 
342  Like all star selectors, the main method is `run`.
343 
344  @section meas_algorithms_objectSizeStarSelector_Config Configuration parameters
345 
346  See @ref ObjectSizeStarSelectorConfig
347 
348  @section meas_algorithms_objectSizeStarSelector_Debug Debug variables
349 
350  ObjectSizeStarSelectorTask has a debug dictionary with the following keys:
351  <dl>
352  <dt>display
353  <dd>bool; if True display debug information
354  <dt>displayExposure
355  <dd>bool; if True display the exposure and spatial cells
356  <dt>plotMagSize
357  <dd>bool: if True display the magnitude-size relation using matplotlib
358  <dt>dumpData
359  <dd>bool; if True dump data to a pickle file
360  </dl>
361 
362  For example, put something like:
363  @code{.py}
364  import lsstDebug
365  def DebugInfo(name):
366  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
367  if name.endswith("objectSizeStarSelector"):
368  di.display = True
369  di.displayExposure = True
370  di.plotMagSize = True
371 
372  return di
373 
374  lsstDebug.Info = DebugInfo
375  @endcode
376  into your `debug.py` file and run your task with the `--debug` flag.
377  """
378  ConfigClass = ObjectSizeStarSelectorConfig
379  usesMatches = False # selectStars does not use its matches argument
380 
381  def selectSources(self, sourceCat, matches=None, exposure=None):
382  """Return a selection of PSF candidates that represent likely stars.
383 
384  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
385 
386  Parameters:
387  -----------
388  sourceCat : `lsst.afw.table.SourceCatalog`
389  Catalog of sources to select from.
390  This catalog must be contiguous in memory.
391  matches : `list` of `lsst.afw.table.ReferenceMatch` or None
392  Ignored in this SourceSelector.
393  exposure : `lsst.afw.image.Exposure` or None
394  The exposure the catalog was built from; used to get the detector
395  to transform to TanPix, and for debug display.
396 
397  Return
398  ------
399  struct : `lsst.pipe.base.Struct`
400  The struct contains the following data:
401 
402  - selected : `array` of `bool``
403  Boolean array of sources that were selected, same length as
404  sourceCat.
405  """
406  import lsstDebug
407  display = lsstDebug.Info(__name__).display
408  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
409  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
410  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
411 
412  detector = None
413  pixToTanPix = None
414  if exposure:
415  detector = exposure.getDetector()
416  if detector:
417  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
418  #
419  # Look at the distribution of stars in the magnitude-size plane
420  #
421  flux = sourceCat.get(self.config.sourceFluxField)
422  fluxErr = sourceCat.get(self.config.sourceFluxField + "Err")
423 
424  xx = numpy.empty(len(sourceCat))
425  xy = numpy.empty_like(xx)
426  yy = numpy.empty_like(xx)
427  for i, source in enumerate(sourceCat):
428  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
429  if pixToTanPix:
430  p = lsst.geom.Point2D(source.getX(), source.getY())
431  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
432  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
433  m.transform(linTransform)
434  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
435 
436  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
437 
438  width = numpy.sqrt(0.5*(xx + yy))
439  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
440  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
441  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
442  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
443  if self.config.doFluxLimit:
444  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
445  if self.config.fluxMax > 0:
446  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
447  if self.config.doSignalToNoiseLimit:
448  bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin)
449  if self.config.signalToNoiseMax > 0:
450  bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax)
451  bad = numpy.logical_or(bad, width < self.config.widthMin)
452  bad = numpy.logical_or(bad, width > self.config.widthMax)
453  good = numpy.logical_not(bad)
454 
455  if not numpy.any(good):
456  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
457 
458  mag = -2.5*numpy.log10(flux[good])
459  width = width[good]
460  #
461  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
462  # the initial peak (of, we presume, stars) from the galaxies
463  #
464  if dumpData:
465  import os
466  import pickle as pickle
467  _ii = 0
468  while True:
469  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
470  if not os.path.exists(pickleFile):
471  break
472  _ii += 1
473 
474  with open(pickleFile, "wb") as fd:
475  pickle.dump(mag, fd, -1)
476  pickle.dump(width, fd, -1)
477 
478  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
479  widthStdAllowed=self.config.widthStdAllowed)
480 
481  if display and plotMagSize:
482  fig = plot(mag, width, centers, clusterId,
483  magType=self.config.sourceFluxField.split(".")[-1].title(),
484  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
485  else:
486  fig = None
487 
488  clusterId = _improveCluster(width, centers, clusterId,
489  nsigma=self.config.nSigmaClip,
490  widthStdAllowed=self.config.widthStdAllowed)
491 
492  if display and plotMagSize:
493  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
494 
495  stellar = (clusterId == 0)
496  #
497  # We know enough to plot, if so requested
498  #
499  frame = 0
500 
501  if fig:
502  if display and displayExposure:
503  disp = afwDisplay.Display(frame=frame)
504  disp.mtv(exposure.getMaskedImage(), title="PSF candidates")
505 
506  global eventHandler
507  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
508  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
509 
510  fig.show()
511 
512  while True:
513  try:
514  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
515  except EOFError:
516  reply = None
517  if not reply:
518  reply = "c"
519 
520  if reply:
521  if reply[0] == "h":
522  print("""\
523  We cluster the points; red are the stellar candidates and the other colours are other clusters.
524  Points labelled + are rejects from the cluster (only for cluster 0).
525 
526  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
527 
528  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the
529  image display.
530  """)
531  elif reply[0] == "p":
532  import pdb
533  pdb.set_trace()
534  elif reply[0] == 'q':
535  sys.exit(1)
536  else:
537  break
538 
539  if display and displayExposure:
540  mi = exposure.getMaskedImage()
541  with disp.Buffering():
542  for i, source in enumerate(sourceCat):
543  if good[i]:
544  ctype = afwDisplay.GREEN # star candidate
545  else:
546  ctype = afwDisplay.RED # not star
547 
548  disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype)
549 
550  # stellar only applies to good==True objects
551  mask = good == True # noqa (numpy bool comparison): E712
552  good[mask] = stellar
553 
554  return Struct(selected=good)
A star selector that looks for a cluster of small objects in a size-magnitude plot.
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)