lsst.meas.algorithms  22.0.0-14-g0da61bd6+74b56c712e
objectSizeStarSelector.py
Go to the documentation of this file.
1 # This file is part of meas_algorithms.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import sys
23 
24 import numpy
25 import warnings
26 from functools import reduce
27 
28 from lsst.log import Log
29 from lsst.pipe.base import Struct
30 import lsst.geom
31 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
32 import lsst.afw.geom as afwGeom
33 import lsst.pex.config as pexConfig
34 import lsst.afw.display as afwDisplay
35 from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry
36 
37 afwDisplay.setDefaultMaskTransparency(75)
38 
39 
40 class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass):
41  doFluxLimit = pexConfig.Field(
42  doc="Apply flux limit to Psf Candidate selection?",
43  dtype=bool,
44  default=True,
45  )
46  fluxMin = pexConfig.Field(
47  doc="specify the minimum psfFlux for good Psf Candidates",
48  dtype=float,
49  default=12500.0,
50  check=lambda x: x >= 0.0,
51  )
52  fluxMax = pexConfig.Field(
53  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
54  dtype=float,
55  default=0.0,
56  check=lambda x: x >= 0.0,
57  )
58  doSignalToNoiseLimit = pexConfig.Field(
59  doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?",
60  dtype=bool,
61  default=False,
62  )
63  signalToNoiseMin = pexConfig.Field(
64  doc="specify the minimum signal-to-noise for good Psf Candidates",
65  dtype=float,
66  default=20.0,
67  check=lambda x: x >= 0.0,
68  )
69  signalToNoiseMax = pexConfig.Field(
70  doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)",
71  dtype=float,
72  default=0.0,
73  check=lambda x: x >= 0.0,
74  )
75  widthMin = pexConfig.Field(
76  doc="minimum width to include in histogram",
77  dtype=float,
78  default=0.0,
79  check=lambda x: x >= 0.0,
80  )
81  widthMax = pexConfig.Field(
82  doc="maximum width to include in histogram",
83  dtype=float,
84  default=10.0,
85  check=lambda x: x >= 0.0,
86  )
87  sourceFluxField = pexConfig.Field(
88  doc="Name of field in Source to use for flux measurement",
89  dtype=str,
90  default="base_GaussianFlux_instFlux",
91  )
92  widthStdAllowed = pexConfig.Field(
93  doc="Standard deviation of width allowed to be interpreted as good stars",
94  dtype=float,
95  default=0.15,
96  check=lambda x: x >= 0.0,
97  )
98  nSigmaClip = pexConfig.Field(
99  doc="Keep objects within this many sigma of cluster 0's median",
100  dtype=float,
101  default=2.0,
102  check=lambda x: x >= 0.0,
103  )
104  badFlags = pexConfig.ListField(
105  doc="List of flags which cause a source to be rejected as bad",
106  dtype=str,
107  default=[
108  "base_PixelFlags_flag_edge",
109  "base_PixelFlags_flag_interpolatedCenter",
110  "base_PixelFlags_flag_saturatedCenter",
111  "base_PixelFlags_flag_crCenter",
112  "base_PixelFlags_flag_bad",
113  "base_PixelFlags_flag_interpolated",
114  ],
115  )
116 
117  def validate(self):
118  BaseSourceSelectorTask.ConfigClass.validate(self)
119  if self.widthMinwidthMin > self.widthMaxwidthMax:
120  msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})"
121  raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg)
122 
123 
125  """A class to handle key strokes with matplotlib displays.
126  """
127 
128  def __init__(self, axes, xs, ys, x, y, frames=[0]):
129  self.axesaxes = axes
130  self.xsxs = xs
131  self.ysys = ys
132  self.xx = x
133  self.yy = y
134  self.framesframes = frames
135 
136  self.cidcid = self.axesaxes.figure.canvas.mpl_connect('key_press_event', self)
137 
138  def __call__(self, ev):
139  if ev.inaxes != self.axesaxes:
140  return
141 
142  if ev.key and ev.key in ("p"):
143  dist = numpy.hypot(self.xsxs - ev.xdata, self.ysys - ev.ydata)
144  dist[numpy.where(numpy.isnan(dist))] = 1e30
145 
146  which = numpy.where(dist == min(dist))
147 
148  x = self.xx[which][0]
149  y = self.yy[which][0]
150  for frame in self.framesframes:
151  disp = afwDisplay.Display(frame=frame)
152  disp.pan(x, y)
153  disp.flush()
154  else:
155  pass
156 
157 
158 def _assignClusters(yvec, centers):
159  """Return a vector of centerIds based on their distance to the centers.
160  """
161  assert len(centers) > 0
162 
163  minDist = numpy.nan*numpy.ones_like(yvec)
164  clusterId = numpy.empty_like(yvec)
165  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
166  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
167  dbl.setLevel(dbl.INFO)
168 
169  # Make sure we are logging aall numpy warnings...
170  oldSettings = numpy.seterr(all="warn")
171  with warnings.catch_warnings(record=True) as w:
172  warnings.simplefilter("always")
173  for i, mean in enumerate(centers):
174  dist = abs(yvec - mean)
175  if i == 0:
176  update = dist == dist # True for all points
177  else:
178  update = dist < minDist
179  if w: # Only do if w is not empty i.e. contains a warning message
180  dbl.trace(str(w[-1]))
181 
182  minDist[update] = dist[update]
183  clusterId[update] = i
184  numpy.seterr(**oldSettings)
185 
186  return clusterId
187 
188 
189 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
190  """A classic k-means algorithm, clustering yvec into nCluster clusters
191 
192  Return the set of centres, and the cluster ID for each of the points
193 
194  If useMedian is true, use the median of the cluster as its centre, rather than
195  the traditional mean
196 
197  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
198  "e.g. why not use the Forgy or random partition initialization methods"
199  however, the approach adopted here seems to work well for the particular sorts of things
200  we're clustering in this application
201  """
202 
203  assert nCluster > 0
204 
205  mean0 = sorted(yvec)[len(yvec)//10] # guess
206  delta = mean0 * widthStdAllowed * 2.0
207  centers = mean0 + delta * numpy.arange(nCluster)
208 
209  func = numpy.median if useMedian else numpy.mean
210 
211  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
212  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
213  while True:
214  oclusterId = clusterId
215  clusterId = _assignClusters(yvec, centers)
216 
217  if numpy.all(clusterId == oclusterId):
218  break
219 
220  for i in range(nCluster):
221  # Only compute func if some points are available; otherwise, default to NaN.
222  pointsInCluster = (clusterId == i)
223  if numpy.any(pointsInCluster):
224  centers[i] = func(yvec[pointsInCluster])
225  else:
226  centers[i] = numpy.nan
227 
228  return centers, clusterId
229 
230 
231 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
232  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median.
233  """
234 
235  nMember = sum(clusterId == clusterNum)
236  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
237  return clusterId
238  for iter in range(nIteration):
239  old_nMember = nMember
240 
241  inCluster0 = clusterId == clusterNum
242  yv = yvec[inCluster0]
243 
244  centers[clusterNum] = numpy.median(yv)
245  stdev = numpy.std(yv)
246 
247  syv = sorted(yv)
248  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
249  median = syv[int(0.5*nMember)]
250 
251  sd = stdev if stdev < stdev_iqr else stdev_iqr
252 
253  if False:
254  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
255  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
256  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
257  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
258 
259  nMember = sum(clusterId == clusterNum)
260  # 'sd < widthStdAllowed * median' prevents too much rejections
261  if nMember == old_nMember or sd < widthStdAllowed * median:
262  break
263 
264  return clusterId
265 
266 
267 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
268  magType="model", clear=True):
269 
270  log = Log.getLogger("objectSizeStarSelector.plot")
271  try:
272  import matplotlib.pyplot as plt
273  except ImportError as e:
274  log.warn("Unable to import matplotlib: %s", e)
275  return
276 
277  try:
278  fig
279  except NameError:
280  fig = plt.figure()
281  else:
282  if clear:
283  fig.clf()
284 
285  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
286 
287  xmin = sorted(mag)[int(0.05*len(mag))]
288  xmax = sorted(mag)[int(0.95*len(mag))]
289 
290  axes.set_xlim(-17.5, -13)
291  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
292  axes.set_ylim(0, 10)
293 
294  colors = ["r", "g", "b", "c", "m", "k", ]
295  for k, mean in enumerate(centers):
296  if k == 0:
297  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
298 
299  li = (clusterId == k)
300  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
301  color=colors[k % len(colors)])
302 
303  li = (clusterId == -1)
304  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
305  color='k')
306 
307  if clear:
308  axes.set_xlabel("Instrumental %s mag" % magType)
309  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
310 
311  return fig
312 
313 
314 @pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry)
316  r"""A star selector that looks for a cluster of small objects in a size-magnitude plot.
317  """
318  ConfigClass = ObjectSizeStarSelectorConfig
319  usesMatches = False # selectStars does not use its matches argument
320 
321  def selectSources(self, sourceCat, matches=None, exposure=None):
322  """Return a selection of PSF candidates that represent likely stars.
323 
324  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
325 
326  Parameters:
327  -----------
328  sourceCat : `lsst.afw.table.SourceCatalog`
329  Catalog of sources to select from.
330  This catalog must be contiguous in memory.
331  matches : `list` of `lsst.afw.table.ReferenceMatch` or None
332  Ignored in this SourceSelector.
333  exposure : `lsst.afw.image.Exposure` or None
334  The exposure the catalog was built from; used to get the detector
335  to transform to TanPix, and for debug display.
336 
337  Return
338  ------
339  struct : `lsst.pipe.base.Struct`
340  The struct contains the following data:
341 
342  - selected : `array` of `bool``
343  Boolean array of sources that were selected, same length as
344  sourceCat.
345  """
346  import lsstDebug
347  display = lsstDebug.Info(__name__).display
348  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
349  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
350  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
351 
352  detector = None
353  pixToTanPix = None
354  if exposure:
355  detector = exposure.getDetector()
356  if detector:
357  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
358  #
359  # Look at the distribution of stars in the magnitude-size plane
360  #
361  flux = sourceCat.get(self.config.sourceFluxField)
362  fluxErr = sourceCat.get(self.config.sourceFluxField + "Err")
363 
364  xx = numpy.empty(len(sourceCat))
365  xy = numpy.empty_like(xx)
366  yy = numpy.empty_like(xx)
367  for i, source in enumerate(sourceCat):
368  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
369  if pixToTanPix:
370  p = lsst.geom.Point2D(source.getX(), source.getY())
371  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
372  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
373  m.transform(linTransform)
374  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
375 
376  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
377 
378  width = numpy.sqrt(0.5*(xx + yy))
379  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
380  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
381  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
382  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
383  if self.config.doFluxLimit:
384  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
385  if self.config.fluxMax > 0:
386  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
387  if self.config.doSignalToNoiseLimit:
388  bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin)
389  if self.config.signalToNoiseMax > 0:
390  bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax)
391  bad = numpy.logical_or(bad, width < self.config.widthMin)
392  bad = numpy.logical_or(bad, width > self.config.widthMax)
393  good = numpy.logical_not(bad)
394 
395  if not numpy.any(good):
396  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
397 
398  mag = -2.5*numpy.log10(flux[good])
399  width = width[good]
400  #
401  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
402  # the initial peak (of, we presume, stars) from the galaxies
403  #
404  if dumpData:
405  import os
406  import pickle as pickle
407  _ii = 0
408  while True:
409  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
410  if not os.path.exists(pickleFile):
411  break
412  _ii += 1
413 
414  with open(pickleFile, "wb") as fd:
415  pickle.dump(mag, fd, -1)
416  pickle.dump(width, fd, -1)
417 
418  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
419  widthStdAllowed=self.config.widthStdAllowed)
420 
421  if display and plotMagSize:
422  fig = plot(mag, width, centers, clusterId,
423  magType=self.config.sourceFluxField.split(".")[-1].title(),
424  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
425  else:
426  fig = None
427 
428  clusterId = _improveCluster(width, centers, clusterId,
429  nsigma=self.config.nSigmaClip,
430  widthStdAllowed=self.config.widthStdAllowed)
431 
432  if display and plotMagSize:
433  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
434 
435  stellar = (clusterId == 0)
436  #
437  # We know enough to plot, if so requested
438  #
439  frame = 0
440 
441  if fig:
442  if display and displayExposure:
443  disp = afwDisplay.Display(frame=frame)
444  disp.mtv(exposure.getMaskedImage(), title="PSF candidates")
445 
446  global eventHandler
447  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
448  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
449 
450  fig.show()
451 
452  while True:
453  try:
454  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
455  except EOFError:
456  reply = None
457  if not reply:
458  reply = "c"
459 
460  if reply:
461  if reply[0] == "h":
462  print("""\
463  We cluster the points; red are the stellar candidates and the other colours are other clusters.
464  Points labelled + are rejects from the cluster (only for cluster 0).
465 
466  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
467 
468  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the
469  image display.
470  """)
471  elif reply[0] == "p":
472  import pdb
473  pdb.set_trace()
474  elif reply[0] == 'q':
475  sys.exit(1)
476  else:
477  break
478 
479  if display and displayExposure:
480  mi = exposure.getMaskedImage()
481  with disp.Buffering():
482  for i, source in enumerate(sourceCat):
483  if good[i]:
484  ctype = afwDisplay.GREEN # star candidate
485  else:
486  ctype = afwDisplay.RED # not star
487 
488  disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype)
489 
490  # stellar only applies to good==True objects
491  mask = good == True # noqa (numpy bool comparison): E712
492  good[mask] = stellar
493 
494  return Struct(selected=good)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)