lsst.meas.algorithms  22.0.1-30-gce6e836d+2b30200b45
objectSizeStarSelector.py
Go to the documentation of this file.
1 # This file is part of meas_algorithms.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 __all__ = ["ObjectSizeStarSelectorConfig", "ObjectSizeStarSelectorTask"]
23 
24 import sys
25 
26 import numpy
27 import warnings
28 from functools import reduce
29 
30 from lsst.utils.logging import getLogger
31 from lsst.pipe.base import Struct
32 import lsst.geom
33 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
34 import lsst.afw.geom as afwGeom
35 import lsst.pex.config as pexConfig
36 import lsst.afw.display as afwDisplay
37 from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry
38 
39 afwDisplay.setDefaultMaskTransparency(75)
40 
41 _LOG = getLogger(__name__)
42 
43 
44 class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass):
45  doFluxLimit = pexConfig.Field(
46  doc="Apply flux limit to Psf Candidate selection?",
47  dtype=bool,
48  default=True,
49  )
50  fluxMin = pexConfig.Field(
51  doc="specify the minimum psfFlux for good Psf Candidates",
52  dtype=float,
53  default=12500.0,
54  check=lambda x: x >= 0.0,
55  )
56  fluxMax = pexConfig.Field(
57  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
58  dtype=float,
59  default=0.0,
60  check=lambda x: x >= 0.0,
61  )
62  doSignalToNoiseLimit = pexConfig.Field(
63  doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?",
64  dtype=bool,
65  default=False,
66  )
67  signalToNoiseMin = pexConfig.Field(
68  doc="specify the minimum signal-to-noise for good Psf Candidates",
69  dtype=float,
70  default=20.0,
71  check=lambda x: x >= 0.0,
72  )
73  signalToNoiseMax = pexConfig.Field(
74  doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)",
75  dtype=float,
76  default=0.0,
77  check=lambda x: x >= 0.0,
78  )
79  widthMin = pexConfig.Field(
80  doc="minimum width to include in histogram",
81  dtype=float,
82  default=0.0,
83  check=lambda x: x >= 0.0,
84  )
85  widthMax = pexConfig.Field(
86  doc="maximum width to include in histogram",
87  dtype=float,
88  default=10.0,
89  check=lambda x: x >= 0.0,
90  )
91  sourceFluxField = pexConfig.Field(
92  doc="Name of field in Source to use for flux measurement",
93  dtype=str,
94  default="base_GaussianFlux_instFlux",
95  )
96  widthStdAllowed = pexConfig.Field(
97  doc="Standard deviation of width allowed to be interpreted as good stars",
98  dtype=float,
99  default=0.15,
100  check=lambda x: x >= 0.0,
101  )
102  nSigmaClip = pexConfig.Field(
103  doc="Keep objects within this many sigma of cluster 0's median",
104  dtype=float,
105  default=2.0,
106  check=lambda x: x >= 0.0,
107  )
108  badFlags = pexConfig.ListField(
109  doc="List of flags which cause a source to be rejected as bad",
110  dtype=str,
111  default=[
112  "base_PixelFlags_flag_edge",
113  "base_PixelFlags_flag_interpolatedCenter",
114  "base_PixelFlags_flag_saturatedCenter",
115  "base_PixelFlags_flag_crCenter",
116  "base_PixelFlags_flag_bad",
117  "base_PixelFlags_flag_interpolated",
118  ],
119  )
120 
121  def validate(self):
122  BaseSourceSelectorTask.ConfigClass.validate(self)
123  if self.widthMinwidthMin > self.widthMaxwidthMax:
124  msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})"
125  raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg)
126 
127 
129  """A class to handle key strokes with matplotlib displays.
130  """
131 
132  def __init__(self, axes, xs, ys, x, y, frames=[0]):
133  self.axesaxes = axes
134  self.xsxs = xs
135  self.ysys = ys
136  self.xx = x
137  self.yy = y
138  self.framesframes = frames
139 
140  self.cidcid = self.axesaxes.figure.canvas.mpl_connect('key_press_event', self)
141 
142  def __call__(self, ev):
143  if ev.inaxes != self.axesaxes:
144  return
145 
146  if ev.key and ev.key in ("p"):
147  dist = numpy.hypot(self.xsxs - ev.xdata, self.ysys - ev.ydata)
148  dist[numpy.where(numpy.isnan(dist))] = 1e30
149 
150  which = numpy.where(dist == min(dist))
151 
152  x = self.xx[which][0]
153  y = self.yy[which][0]
154  for frame in self.framesframes:
155  disp = afwDisplay.Display(frame=frame)
156  disp.pan(x, y)
157  disp.flush()
158  else:
159  pass
160 
161 
162 def _assignClusters(yvec, centers):
163  """Return a vector of centerIds based on their distance to the centers.
164  """
165  assert len(centers) > 0
166 
167  minDist = numpy.nan*numpy.ones_like(yvec)
168  clusterId = numpy.empty_like(yvec)
169  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
170  dbl = _LOG.getChild("_assignClusters")
171  dbl.setLevel(dbl.INFO)
172 
173  # Make sure we are logging aall numpy warnings...
174  oldSettings = numpy.seterr(all="warn")
175  with warnings.catch_warnings(record=True) as w:
176  warnings.simplefilter("always")
177  for i, mean in enumerate(centers):
178  dist = abs(yvec - mean)
179  if i == 0:
180  update = dist == dist # True for all points
181  else:
182  update = dist < minDist
183  if w: # Only do if w is not empty i.e. contains a warning message
184  dbl.trace(str(w[-1]))
185 
186  minDist[update] = dist[update]
187  clusterId[update] = i
188  numpy.seterr(**oldSettings)
189 
190  return clusterId
191 
192 
193 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
194  """A classic k-means algorithm, clustering yvec into nCluster clusters
195 
196  Return the set of centres, and the cluster ID for each of the points
197 
198  If useMedian is true, use the median of the cluster as its centre, rather than
199  the traditional mean
200 
201  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
202  "e.g. why not use the Forgy or random partition initialization methods"
203  however, the approach adopted here seems to work well for the particular sorts of things
204  we're clustering in this application
205  """
206 
207  assert nCluster > 0
208 
209  mean0 = sorted(yvec)[len(yvec)//10] # guess
210  delta = mean0 * widthStdAllowed * 2.0
211  centers = mean0 + delta * numpy.arange(nCluster)
212 
213  func = numpy.median if useMedian else numpy.mean
214 
215  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
216  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
217  while True:
218  oclusterId = clusterId
219  clusterId = _assignClusters(yvec, centers)
220 
221  if numpy.all(clusterId == oclusterId):
222  break
223 
224  for i in range(nCluster):
225  # Only compute func if some points are available; otherwise, default to NaN.
226  pointsInCluster = (clusterId == i)
227  if numpy.any(pointsInCluster):
228  centers[i] = func(yvec[pointsInCluster])
229  else:
230  centers[i] = numpy.nan
231 
232  return centers, clusterId
233 
234 
235 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
236  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median.
237  """
238 
239  nMember = sum(clusterId == clusterNum)
240  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
241  return clusterId
242  for iter in range(nIteration):
243  old_nMember = nMember
244 
245  inCluster0 = clusterId == clusterNum
246  yv = yvec[inCluster0]
247 
248  centers[clusterNum] = numpy.median(yv)
249  stdev = numpy.std(yv)
250 
251  syv = sorted(yv)
252  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
253  median = syv[int(0.5*nMember)]
254 
255  sd = stdev if stdev < stdev_iqr else stdev_iqr
256 
257  if False:
258  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
259  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
260  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
261  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
262 
263  nMember = sum(clusterId == clusterNum)
264  # 'sd < widthStdAllowed * median' prevents too much rejections
265  if nMember == old_nMember or sd < widthStdAllowed * median:
266  break
267 
268  return clusterId
269 
270 
271 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
272  magType="model", clear=True):
273 
274  log = _LOG.getChild("plot")
275  try:
276  import matplotlib.pyplot as plt
277  except ImportError as e:
278  log.warning("Unable to import matplotlib: %s", e)
279  return
280 
281  try:
282  fig
283  except NameError:
284  fig = plt.figure()
285  else:
286  if clear:
287  fig.clf()
288 
289  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
290 
291  xmin = sorted(mag)[int(0.05*len(mag))]
292  xmax = sorted(mag)[int(0.95*len(mag))]
293 
294  axes.set_xlim(-17.5, -13)
295  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
296  axes.set_ylim(0, 10)
297 
298  colors = ["r", "g", "b", "c", "m", "k", ]
299  for k, mean in enumerate(centers):
300  if k == 0:
301  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
302 
303  li = (clusterId == k)
304  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
305  color=colors[k % len(colors)])
306 
307  li = (clusterId == -1)
308  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
309  color='k')
310 
311  if clear:
312  axes.set_xlabel("Instrumental %s mag" % magType)
313  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
314 
315  return fig
316 
317 
318 @pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry)
320  r"""A star selector that looks for a cluster of small objects in a size-magnitude plot.
321  """
322  ConfigClass = ObjectSizeStarSelectorConfig
323  usesMatches = False # selectStars does not use its matches argument
324 
325  def selectSources(self, sourceCat, matches=None, exposure=None):
326  """Return a selection of PSF candidates that represent likely stars.
327 
328  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
329 
330  Parameters:
331  -----------
332  sourceCat : `lsst.afw.table.SourceCatalog`
333  Catalog of sources to select from.
334  This catalog must be contiguous in memory.
335  matches : `list` of `lsst.afw.table.ReferenceMatch` or None
336  Ignored in this SourceSelector.
337  exposure : `lsst.afw.image.Exposure` or None
338  The exposure the catalog was built from; used to get the detector
339  to transform to TanPix, and for debug display.
340 
341  Return
342  ------
343  struct : `lsst.pipe.base.Struct`
344  The struct contains the following data:
345 
346  - selected : `array` of `bool``
347  Boolean array of sources that were selected, same length as
348  sourceCat.
349  """
350  import lsstDebug
351  display = lsstDebug.Info(__name__).display
352  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
353  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
354  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
355 
356  detector = None
357  pixToTanPix = None
358  if exposure:
359  detector = exposure.getDetector()
360  if detector:
361  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
362  #
363  # Look at the distribution of stars in the magnitude-size plane
364  #
365  flux = sourceCat.get(self.config.sourceFluxField)
366  fluxErr = sourceCat.get(self.config.sourceFluxField + "Err")
367 
368  xx = numpy.empty(len(sourceCat))
369  xy = numpy.empty_like(xx)
370  yy = numpy.empty_like(xx)
371  for i, source in enumerate(sourceCat):
372  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
373  if pixToTanPix:
374  p = lsst.geom.Point2D(source.getX(), source.getY())
375  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
376  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
377  m.transform(linTransform)
378  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
379 
380  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
381 
382  width = numpy.sqrt(0.5*(xx + yy))
383  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
384  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
385  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
386  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
387  if self.config.doFluxLimit:
388  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
389  if self.config.fluxMax > 0:
390  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
391  if self.config.doSignalToNoiseLimit:
392  bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin)
393  if self.config.signalToNoiseMax > 0:
394  bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax)
395  bad = numpy.logical_or(bad, width < self.config.widthMin)
396  bad = numpy.logical_or(bad, width > self.config.widthMax)
397  good = numpy.logical_not(bad)
398 
399  if not numpy.any(good):
400  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
401 
402  mag = -2.5*numpy.log10(flux[good])
403  width = width[good]
404  #
405  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
406  # the initial peak (of, we presume, stars) from the galaxies
407  #
408  if dumpData:
409  import os
410  import pickle as pickle
411  _ii = 0
412  while True:
413  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
414  if not os.path.exists(pickleFile):
415  break
416  _ii += 1
417 
418  with open(pickleFile, "wb") as fd:
419  pickle.dump(mag, fd, -1)
420  pickle.dump(width, fd, -1)
421 
422  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
423  widthStdAllowed=self.config.widthStdAllowed)
424 
425  if display and plotMagSize:
426  fig = plot(mag, width, centers, clusterId,
427  magType=self.config.sourceFluxField.split(".")[-1].title(),
428  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
429  else:
430  fig = None
431 
432  clusterId = _improveCluster(width, centers, clusterId,
433  nsigma=self.config.nSigmaClip,
434  widthStdAllowed=self.config.widthStdAllowed)
435 
436  if display and plotMagSize:
437  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
438 
439  stellar = (clusterId == 0)
440  #
441  # We know enough to plot, if so requested
442  #
443  frame = 0
444 
445  if fig:
446  if display and displayExposure:
447  disp = afwDisplay.Display(frame=frame)
448  disp.mtv(exposure.getMaskedImage(), title="PSF candidates")
449 
450  global eventHandler
451  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
452  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
453 
454  fig.show()
455 
456  while True:
457  try:
458  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
459  except EOFError:
460  reply = None
461  if not reply:
462  reply = "c"
463 
464  if reply:
465  if reply[0] == "h":
466  print("""\
467  We cluster the points; red are the stellar candidates and the other colours are other clusters.
468  Points labelled + are rejects from the cluster (only for cluster 0).
469 
470  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
471 
472  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the
473  image display.
474  """)
475  elif reply[0] == "p":
476  import pdb
477  pdb.set_trace()
478  elif reply[0] == 'q':
479  sys.exit(1)
480  else:
481  break
482 
483  if display and displayExposure:
484  mi = exposure.getMaskedImage()
485  with disp.Buffering():
486  for i, source in enumerate(sourceCat):
487  if good[i]:
488  ctype = afwDisplay.GREEN # star candidate
489  else:
490  ctype = afwDisplay.RED # not star
491 
492  disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype)
493 
494  # stellar only applies to good==True objects
495  mask = good == True # noqa (numpy bool comparison): E712
496  good[mask] = stellar
497 
498  return Struct(selected=good)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)