lsst.meas.algorithms  22.0.1-23-gc2439a9a+e37b83d4fa
objectSizeStarSelector.py
Go to the documentation of this file.
1 # This file is part of meas_algorithms.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 __all__ = ["ObjectSizeStarSelectorConfig", "ObjectSizeStarSelectorTask"]
23 
24 import sys
25 
26 import numpy
27 import warnings
28 from functools import reduce
29 
30 from lsst.log import Log
31 from lsst.pipe.base import Struct
32 import lsst.geom
33 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
34 import lsst.afw.geom as afwGeom
35 import lsst.pex.config as pexConfig
36 import lsst.afw.display as afwDisplay
37 from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry
38 
39 afwDisplay.setDefaultMaskTransparency(75)
40 
41 
42 class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass):
43  doFluxLimit = pexConfig.Field(
44  doc="Apply flux limit to Psf Candidate selection?",
45  dtype=bool,
46  default=True,
47  )
48  fluxMin = pexConfig.Field(
49  doc="specify the minimum psfFlux for good Psf Candidates",
50  dtype=float,
51  default=12500.0,
52  check=lambda x: x >= 0.0,
53  )
54  fluxMax = pexConfig.Field(
55  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
56  dtype=float,
57  default=0.0,
58  check=lambda x: x >= 0.0,
59  )
60  doSignalToNoiseLimit = pexConfig.Field(
61  doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?",
62  dtype=bool,
63  default=False,
64  )
65  signalToNoiseMin = pexConfig.Field(
66  doc="specify the minimum signal-to-noise for good Psf Candidates",
67  dtype=float,
68  default=20.0,
69  check=lambda x: x >= 0.0,
70  )
71  signalToNoiseMax = pexConfig.Field(
72  doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)",
73  dtype=float,
74  default=0.0,
75  check=lambda x: x >= 0.0,
76  )
77  widthMin = pexConfig.Field(
78  doc="minimum width to include in histogram",
79  dtype=float,
80  default=0.0,
81  check=lambda x: x >= 0.0,
82  )
83  widthMax = pexConfig.Field(
84  doc="maximum width to include in histogram",
85  dtype=float,
86  default=10.0,
87  check=lambda x: x >= 0.0,
88  )
89  sourceFluxField = pexConfig.Field(
90  doc="Name of field in Source to use for flux measurement",
91  dtype=str,
92  default="base_GaussianFlux_instFlux",
93  )
94  widthStdAllowed = pexConfig.Field(
95  doc="Standard deviation of width allowed to be interpreted as good stars",
96  dtype=float,
97  default=0.15,
98  check=lambda x: x >= 0.0,
99  )
100  nSigmaClip = pexConfig.Field(
101  doc="Keep objects within this many sigma of cluster 0's median",
102  dtype=float,
103  default=2.0,
104  check=lambda x: x >= 0.0,
105  )
106  badFlags = pexConfig.ListField(
107  doc="List of flags which cause a source to be rejected as bad",
108  dtype=str,
109  default=[
110  "base_PixelFlags_flag_edge",
111  "base_PixelFlags_flag_interpolatedCenter",
112  "base_PixelFlags_flag_saturatedCenter",
113  "base_PixelFlags_flag_crCenter",
114  "base_PixelFlags_flag_bad",
115  "base_PixelFlags_flag_interpolated",
116  ],
117  )
118 
119  def validate(self):
120  BaseSourceSelectorTask.ConfigClass.validate(self)
121  if self.widthMinwidthMin > self.widthMaxwidthMax:
122  msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})"
123  raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg)
124 
125 
127  """A class to handle key strokes with matplotlib displays.
128  """
129 
130  def __init__(self, axes, xs, ys, x, y, frames=[0]):
131  self.axesaxes = axes
132  self.xsxs = xs
133  self.ysys = ys
134  self.xx = x
135  self.yy = y
136  self.framesframes = frames
137 
138  self.cidcid = self.axesaxes.figure.canvas.mpl_connect('key_press_event', self)
139 
140  def __call__(self, ev):
141  if ev.inaxes != self.axesaxes:
142  return
143 
144  if ev.key and ev.key in ("p"):
145  dist = numpy.hypot(self.xsxs - ev.xdata, self.ysys - ev.ydata)
146  dist[numpy.where(numpy.isnan(dist))] = 1e30
147 
148  which = numpy.where(dist == min(dist))
149 
150  x = self.xx[which][0]
151  y = self.yy[which][0]
152  for frame in self.framesframes:
153  disp = afwDisplay.Display(frame=frame)
154  disp.pan(x, y)
155  disp.flush()
156  else:
157  pass
158 
159 
160 def _assignClusters(yvec, centers):
161  """Return a vector of centerIds based on their distance to the centers.
162  """
163  assert len(centers) > 0
164 
165  minDist = numpy.nan*numpy.ones_like(yvec)
166  clusterId = numpy.empty_like(yvec)
167  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
168  dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
169  dbl.setLevel(dbl.INFO)
170 
171  # Make sure we are logging aall numpy warnings...
172  oldSettings = numpy.seterr(all="warn")
173  with warnings.catch_warnings(record=True) as w:
174  warnings.simplefilter("always")
175  for i, mean in enumerate(centers):
176  dist = abs(yvec - mean)
177  if i == 0:
178  update = dist == dist # True for all points
179  else:
180  update = dist < minDist
181  if w: # Only do if w is not empty i.e. contains a warning message
182  dbl.trace(str(w[-1]))
183 
184  minDist[update] = dist[update]
185  clusterId[update] = i
186  numpy.seterr(**oldSettings)
187 
188  return clusterId
189 
190 
191 def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
192  """A classic k-means algorithm, clustering yvec into nCluster clusters
193 
194  Return the set of centres, and the cluster ID for each of the points
195 
196  If useMedian is true, use the median of the cluster as its centre, rather than
197  the traditional mean
198 
199  Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
200  "e.g. why not use the Forgy or random partition initialization methods"
201  however, the approach adopted here seems to work well for the particular sorts of things
202  we're clustering in this application
203  """
204 
205  assert nCluster > 0
206 
207  mean0 = sorted(yvec)[len(yvec)//10] # guess
208  delta = mean0 * widthStdAllowed * 2.0
209  centers = mean0 + delta * numpy.arange(nCluster)
210 
211  func = numpy.median if useMedian else numpy.mean
212 
213  clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
214  clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
215  while True:
216  oclusterId = clusterId
217  clusterId = _assignClusters(yvec, centers)
218 
219  if numpy.all(clusterId == oclusterId):
220  break
221 
222  for i in range(nCluster):
223  # Only compute func if some points are available; otherwise, default to NaN.
224  pointsInCluster = (clusterId == i)
225  if numpy.any(pointsInCluster):
226  centers[i] = func(yvec[pointsInCluster])
227  else:
228  centers[i] = numpy.nan
229 
230  return centers, clusterId
231 
232 
233 def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
234  """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median.
235  """
236 
237  nMember = sum(clusterId == clusterNum)
238  if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
239  return clusterId
240  for iter in range(nIteration):
241  old_nMember = nMember
242 
243  inCluster0 = clusterId == clusterNum
244  yv = yvec[inCluster0]
245 
246  centers[clusterNum] = numpy.median(yv)
247  stdev = numpy.std(yv)
248 
249  syv = sorted(yv)
250  stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
251  median = syv[int(0.5*nMember)]
252 
253  sd = stdev if stdev < stdev_iqr else stdev_iqr
254 
255  if False:
256  print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
257  newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
258  clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
259  clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
260 
261  nMember = sum(clusterId == clusterNum)
262  # 'sd < widthStdAllowed * median' prevents too much rejections
263  if nMember == old_nMember or sd < widthStdAllowed * median:
264  break
265 
266  return clusterId
267 
268 
269 def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
270  magType="model", clear=True):
271 
272  log = Log.getLogger("objectSizeStarSelector.plot")
273  try:
274  import matplotlib.pyplot as plt
275  except ImportError as e:
276  log.warning("Unable to import matplotlib: %s", e)
277  return
278 
279  try:
280  fig
281  except NameError:
282  fig = plt.figure()
283  else:
284  if clear:
285  fig.clf()
286 
287  axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
288 
289  xmin = sorted(mag)[int(0.05*len(mag))]
290  xmax = sorted(mag)[int(0.95*len(mag))]
291 
292  axes.set_xlim(-17.5, -13)
293  axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
294  axes.set_ylim(0, 10)
295 
296  colors = ["r", "g", "b", "c", "m", "k", ]
297  for k, mean in enumerate(centers):
298  if k == 0:
299  axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
300 
301  li = (clusterId == k)
302  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
303  color=colors[k % len(colors)])
304 
305  li = (clusterId == -1)
306  axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
307  color='k')
308 
309  if clear:
310  axes.set_xlabel("Instrumental %s mag" % magType)
311  axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
312 
313  return fig
314 
315 
316 @pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry)
318  r"""A star selector that looks for a cluster of small objects in a size-magnitude plot.
319  """
320  ConfigClass = ObjectSizeStarSelectorConfig
321  usesMatches = False # selectStars does not use its matches argument
322 
323  def selectSources(self, sourceCat, matches=None, exposure=None):
324  """Return a selection of PSF candidates that represent likely stars.
325 
326  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
327 
328  Parameters:
329  -----------
330  sourceCat : `lsst.afw.table.SourceCatalog`
331  Catalog of sources to select from.
332  This catalog must be contiguous in memory.
333  matches : `list` of `lsst.afw.table.ReferenceMatch` or None
334  Ignored in this SourceSelector.
335  exposure : `lsst.afw.image.Exposure` or None
336  The exposure the catalog was built from; used to get the detector
337  to transform to TanPix, and for debug display.
338 
339  Return
340  ------
341  struct : `lsst.pipe.base.Struct`
342  The struct contains the following data:
343 
344  - selected : `array` of `bool``
345  Boolean array of sources that were selected, same length as
346  sourceCat.
347  """
348  import lsstDebug
349  display = lsstDebug.Info(__name__).display
350  displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
351  plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
352  dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
353 
354  detector = None
355  pixToTanPix = None
356  if exposure:
357  detector = exposure.getDetector()
358  if detector:
359  pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
360  #
361  # Look at the distribution of stars in the magnitude-size plane
362  #
363  flux = sourceCat.get(self.config.sourceFluxField)
364  fluxErr = sourceCat.get(self.config.sourceFluxField + "Err")
365 
366  xx = numpy.empty(len(sourceCat))
367  xy = numpy.empty_like(xx)
368  yy = numpy.empty_like(xx)
369  for i, source in enumerate(sourceCat):
370  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
371  if pixToTanPix:
372  p = lsst.geom.Point2D(source.getX(), source.getY())
373  linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
374  m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
375  m.transform(linTransform)
376  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
377 
378  xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
379 
380  width = numpy.sqrt(0.5*(xx + yy))
381  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
382  bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
383  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
384  bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
385  if self.config.doFluxLimit:
386  bad = numpy.logical_or(bad, flux < self.config.fluxMin)
387  if self.config.fluxMax > 0:
388  bad = numpy.logical_or(bad, flux > self.config.fluxMax)
389  if self.config.doSignalToNoiseLimit:
390  bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin)
391  if self.config.signalToNoiseMax > 0:
392  bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax)
393  bad = numpy.logical_or(bad, width < self.config.widthMin)
394  bad = numpy.logical_or(bad, width > self.config.widthMax)
395  good = numpy.logical_not(bad)
396 
397  if not numpy.any(good):
398  raise RuntimeError("No objects passed our cuts for consideration as psf stars")
399 
400  mag = -2.5*numpy.log10(flux[good])
401  width = width[good]
402  #
403  # Look for the maximum in the size histogram, then search upwards for the minimum that separates
404  # the initial peak (of, we presume, stars) from the galaxies
405  #
406  if dumpData:
407  import os
408  import pickle as pickle
409  _ii = 0
410  while True:
411  pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
412  if not os.path.exists(pickleFile):
413  break
414  _ii += 1
415 
416  with open(pickleFile, "wb") as fd:
417  pickle.dump(mag, fd, -1)
418  pickle.dump(width, fd, -1)
419 
420  centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
421  widthStdAllowed=self.config.widthStdAllowed)
422 
423  if display and plotMagSize:
424  fig = plot(mag, width, centers, clusterId,
425  magType=self.config.sourceFluxField.split(".")[-1].title(),
426  marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
427  else:
428  fig = None
429 
430  clusterId = _improveCluster(width, centers, clusterId,
431  nsigma=self.config.nSigmaClip,
432  widthStdAllowed=self.config.widthStdAllowed)
433 
434  if display and plotMagSize:
435  plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
436 
437  stellar = (clusterId == 0)
438  #
439  # We know enough to plot, if so requested
440  #
441  frame = 0
442 
443  if fig:
444  if display and displayExposure:
445  disp = afwDisplay.Display(frame=frame)
446  disp.mtv(exposure.getMaskedImage(), title="PSF candidates")
447 
448  global eventHandler
449  eventHandler = EventHandler(fig.get_axes()[0], mag, width,
450  sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
451 
452  fig.show()
453 
454  while True:
455  try:
456  reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
457  except EOFError:
458  reply = None
459  if not reply:
460  reply = "c"
461 
462  if reply:
463  if reply[0] == "h":
464  print("""\
465  We cluster the points; red are the stellar candidates and the other colours are other clusters.
466  Points labelled + are rejects from the cluster (only for cluster 0).
467 
468  At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
469 
470  If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the
471  image display.
472  """)
473  elif reply[0] == "p":
474  import pdb
475  pdb.set_trace()
476  elif reply[0] == 'q':
477  sys.exit(1)
478  else:
479  break
480 
481  if display and displayExposure:
482  mi = exposure.getMaskedImage()
483  with disp.Buffering():
484  for i, source in enumerate(sourceCat):
485  if good[i]:
486  ctype = afwDisplay.GREEN # star candidate
487  else:
488  ctype = afwDisplay.RED # not star
489 
490  disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype)
491 
492  # stellar only applies to good==True objects
493  mask = good == True # noqa (numpy bool comparison): E712
494  good[mask] = stellar
495 
496  return Struct(selected=good)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)