lsst.meas.algorithms 23.0.0+47ff9911fb
objectSizeStarSelector.py
Go to the documentation of this file.
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["ObjectSizeStarSelectorConfig", "ObjectSizeStarSelectorTask"]
23
24import sys
25
26import numpy
27import warnings
28from functools import reduce
29
30from lsst.log import Log
31from lsst.pipe.base import Struct
32import lsst.geom
33from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
34import lsst.afw.geom as afwGeom
35import lsst.pex.config as pexConfig
36import lsst.afw.display as afwDisplay
37from .sourceSelector import BaseSourceSelectorTask, sourceSelectorRegistry
38
39afwDisplay.setDefaultMaskTransparency(75)
40
41
42class ObjectSizeStarSelectorConfig(BaseSourceSelectorTask.ConfigClass):
43 doFluxLimit = pexConfig.Field(
44 doc="Apply flux limit to Psf Candidate selection?",
45 dtype=bool,
46 default=True,
47 )
48 fluxMin = pexConfig.Field(
49 doc="specify the minimum psfFlux for good Psf Candidates",
50 dtype=float,
51 default=12500.0,
52 check=lambda x: x >= 0.0,
53 )
54 fluxMax = pexConfig.Field(
55 doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
56 dtype=float,
57 default=0.0,
58 check=lambda x: x >= 0.0,
59 )
60 doSignalToNoiseLimit = pexConfig.Field(
61 doc="Apply signal-to-noise (i.e. flux/fluxErr) limit to Psf Candidate selection?",
62 dtype=bool,
63 default=False,
64 )
65 # Note that the current default is conditioned on the detection thresholds
66 # set in the characterizeImage setDefaults function for the measurePsf
67 # stage.
68 signalToNoiseMin = pexConfig.Field(
69 doc="specify the minimum signal-to-noise for good Psf Candidates "
70 "(value should take into consideration the detection thresholds "
71 "set for the catalog of interest)",
72 dtype=float,
73 default=50.0,
74 check=lambda x: x >= 0.0,
75 )
76 signalToNoiseMax = pexConfig.Field(
77 doc="specify the maximum signal-to-noise for good Psf Candidates (ignored if == 0)",
78 dtype=float,
79 default=0.0,
80 check=lambda x: x >= 0.0,
81 )
82 widthMin = pexConfig.Field(
83 doc="minimum width to include in histogram",
84 dtype=float,
85 default=0.0,
86 check=lambda x: x >= 0.0,
87 )
88 widthMax = pexConfig.Field(
89 doc="maximum width to include in histogram",
90 dtype=float,
91 default=10.0,
92 check=lambda x: x >= 0.0,
93 )
94 sourceFluxField = pexConfig.Field(
95 doc="Name of field in Source to use for flux measurement",
96 dtype=str,
97 default="base_GaussianFlux_instFlux",
98 )
99 widthStdAllowed = pexConfig.Field(
100 doc="Standard deviation of width allowed to be interpreted as good stars",
101 dtype=float,
102 default=0.15,
103 check=lambda x: x >= 0.0,
104 )
105 nSigmaClip = pexConfig.Field(
106 doc="Keep objects within this many sigma of cluster 0's median",
107 dtype=float,
108 default=2.0,
109 check=lambda x: x >= 0.0,
110 )
111 badFlags = pexConfig.ListField(
112 doc="List of flags which cause a source to be rejected as bad",
113 dtype=str,
114 default=[
115 "base_PixelFlags_flag_edge",
116 "base_PixelFlags_flag_interpolatedCenter",
117 "base_PixelFlags_flag_saturatedCenter",
118 "base_PixelFlags_flag_crCenter",
119 "base_PixelFlags_flag_bad",
120 "base_PixelFlags_flag_interpolated",
121 ],
122 )
123
124 def validate(self):
125 BaseSourceSelectorTask.ConfigClass.validate(self)
126 if self.widthMinwidthMin > self.widthMaxwidthMax:
127 msg = f"widthMin ({self.widthMin}) > widthMax ({self.widthMax})"
128 raise pexConfig.FieldValidationError(ObjectSizeStarSelectorConfig.widthMin, self, msg)
129
130
132 """A class to handle key strokes with matplotlib displays.
133 """
134
135 def __init__(self, axes, xs, ys, x, y, frames=[0]):
136 self.axesaxes = axes
137 self.xsxs = xs
138 self.ysys = ys
139 self.xx = x
140 self.yy = y
141 self.framesframes = frames
142
143 self.cidcid = self.axesaxes.figure.canvas.mpl_connect('key_press_event', self)
144
145 def __call__(self, ev):
146 if ev.inaxes != self.axesaxes:
147 return
148
149 if ev.key and ev.key in ("p"):
150 dist = numpy.hypot(self.xsxs - ev.xdata, self.ysys - ev.ydata)
151 dist[numpy.where(numpy.isnan(dist))] = 1e30
152
153 which = numpy.where(dist == min(dist))
154
155 x = self.xx[which][0]
156 y = self.yy[which][0]
157 for frame in self.framesframes:
158 disp = afwDisplay.Display(frame=frame)
159 disp.pan(x, y)
160 disp.flush()
161 else:
162 pass
163
164
165def _assignClusters(yvec, centers):
166 """Return a vector of centerIds based on their distance to the centers.
167 """
168 assert len(centers) > 0
169
170 minDist = numpy.nan*numpy.ones_like(yvec)
171 clusterId = numpy.empty_like(yvec)
172 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
173 dbl = Log.getLogger("objectSizeStarSelector._assignClusters")
174 dbl.setLevel(dbl.INFO)
175
176 # Make sure we are logging aall numpy warnings...
177 oldSettings = numpy.seterr(all="warn")
178 with warnings.catch_warnings(record=True) as w:
179 warnings.simplefilter("always")
180 for i, mean in enumerate(centers):
181 dist = abs(yvec - mean)
182 if i == 0:
183 update = dist == dist # True for all points
184 else:
185 update = dist < minDist
186 if w: # Only do if w is not empty i.e. contains a warning message
187 dbl.trace(str(w[-1]))
188
189 minDist[update] = dist[update]
190 clusterId[update] = i
191 numpy.seterr(**oldSettings)
192
193 return clusterId
194
195
196def _kcenters(yvec, nCluster, useMedian=False, widthStdAllowed=0.15):
197 """A classic k-means algorithm, clustering yvec into nCluster clusters
198
199 Return the set of centres, and the cluster ID for each of the points
200
201 If useMedian is true, use the median of the cluster as its centre, rather than
202 the traditional mean
203
204 Serge Monkewitz points out that there other (maybe smarter) ways of seeding the means:
205 "e.g. why not use the Forgy or random partition initialization methods"
206 however, the approach adopted here seems to work well for the particular sorts of things
207 we're clustering in this application
208 """
209
210 assert nCluster > 0
211
212 mean0 = sorted(yvec)[len(yvec)//10] # guess
213 delta = mean0 * widthStdAllowed * 2.0
214 centers = mean0 + delta * numpy.arange(nCluster)
215
216 func = numpy.median if useMedian else numpy.mean
217
218 clusterId = numpy.zeros_like(yvec) - 1 # which cluster the points are assigned to
219 clusterId.dtype = int # zeros_like(..., dtype=int) isn't in numpy 1.5
220 while True:
221 oclusterId = clusterId
222 clusterId = _assignClusters(yvec, centers)
223
224 if numpy.all(clusterId == oclusterId):
225 break
226
227 for i in range(nCluster):
228 # Only compute func if some points are available; otherwise, default to NaN.
229 pointsInCluster = (clusterId == i)
230 if numpy.any(pointsInCluster):
231 centers[i] = func(yvec[pointsInCluster])
232 else:
233 centers[i] = numpy.nan
234
235 return centers, clusterId
236
237
238def _improveCluster(yvec, centers, clusterId, nsigma=2.0, nIteration=10, clusterNum=0, widthStdAllowed=0.15):
239 """Improve our estimate of one of the clusters (clusterNum) by sigma-clipping around its median.
240 """
241
242 nMember = sum(clusterId == clusterNum)
243 if nMember < 5: # can't compute meaningful interquartile range, so no chance of improvement
244 return clusterId
245 for iter in range(nIteration):
246 old_nMember = nMember
247
248 inCluster0 = clusterId == clusterNum
249 yv = yvec[inCluster0]
250
251 centers[clusterNum] = numpy.median(yv)
252 stdev = numpy.std(yv)
253
254 syv = sorted(yv)
255 stdev_iqr = 0.741*(syv[int(0.75*nMember)] - syv[int(0.25*nMember)])
256 median = syv[int(0.5*nMember)]
257
258 sd = stdev if stdev < stdev_iqr else stdev_iqr
259
260 if False:
261 print("sigma(iqr) = %.3f, sigma = %.3f" % (stdev_iqr, numpy.std(yv)))
262 newCluster0 = abs(yvec - centers[clusterNum]) < nsigma*sd
263 clusterId[numpy.logical_and(inCluster0, newCluster0)] = clusterNum
264 clusterId[numpy.logical_and(inCluster0, numpy.logical_not(newCluster0))] = -1
265
266 nMember = sum(clusterId == clusterNum)
267 # 'sd < widthStdAllowed * median' prevents too much rejections
268 if nMember == old_nMember or sd < widthStdAllowed * median:
269 break
270
271 return clusterId
272
273
274def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-',
275 magType="model", clear=True):
276
277 log = Log.getLogger("objectSizeStarSelector.plot")
278 try:
279 import matplotlib.pyplot as plt
280 except ImportError as e:
281 log.warning("Unable to import matplotlib: %s", e)
282 return
283
284 try:
285 fig
286 except NameError:
287 fig = plt.figure()
288 else:
289 if clear:
290 fig.clf()
291
292 axes = fig.add_axes((0.1, 0.1, 0.85, 0.80))
293
294 xmin = sorted(mag)[int(0.05*len(mag))]
295 xmax = sorted(mag)[int(0.95*len(mag))]
296
297 axes.set_xlim(-17.5, -13)
298 axes.set_xlim(xmin - 0.1*(xmax - xmin), xmax + 0.1*(xmax - xmin))
299 axes.set_ylim(0, 10)
300
301 colors = ["r", "g", "b", "c", "m", "k", ]
302 for k, mean in enumerate(centers):
303 if k == 0:
304 axes.plot(axes.get_xlim(), (mean, mean,), "k%s" % ltype)
305
306 li = (clusterId == k)
307 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
308 color=colors[k % len(colors)])
309
310 li = (clusterId == -1)
311 axes.plot(mag[li], width[li], marker, markersize=markersize, markeredgewidth=markeredgewidth,
312 color='k')
313
314 if clear:
315 axes.set_xlabel("Instrumental %s mag" % magType)
316 axes.set_ylabel(r"$\sqrt{(I_{xx} + I_{yy})/2}$")
317
318 return fig
319
320
321@pexConfig.registerConfigurable("objectSize", sourceSelectorRegistry)
323 r"""A star selector that looks for a cluster of small objects in a size-magnitude plot.
324 """
325 ConfigClass = ObjectSizeStarSelectorConfig
326 usesMatches = False # selectStars does not use its matches argument
327
328 def selectSources(self, sourceCat, matches=None, exposure=None):
329 """Return a selection of PSF candidates that represent likely stars.
330
331 A list of PSF candidates may be used by a PSF fitter to construct a PSF.
332
333 Parameters:
334 -----------
335 sourceCat : `lsst.afw.table.SourceCatalog`
336 Catalog of sources to select from.
337 This catalog must be contiguous in memory.
338 matches : `list` of `lsst.afw.table.ReferenceMatch` or None
339 Ignored in this SourceSelector.
340 exposure : `lsst.afw.image.Exposure` or None
341 The exposure the catalog was built from; used to get the detector
342 to transform to TanPix, and for debug display.
343
344 Return
345 ------
346 struct : `lsst.pipe.base.Struct`
347 The struct contains the following data:
348
349 - selected : `array` of `bool``
350 Boolean array of sources that were selected, same length as
351 sourceCat.
352 """
353 import lsstDebug
354 display = lsstDebug.Info(__name__).display
355 displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
356 plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
357 dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
358
359 detector = None
360 pixToTanPix = None
361 if exposure:
362 detector = exposure.getDetector()
363 if detector:
364 pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
365 #
366 # Look at the distribution of stars in the magnitude-size plane
367 #
368 flux = sourceCat.get(self.config.sourceFluxField)
369 fluxErr = sourceCat.get(self.config.sourceFluxField + "Err")
370
371 xx = numpy.empty(len(sourceCat))
372 xy = numpy.empty_like(xx)
373 yy = numpy.empty_like(xx)
374 for i, source in enumerate(sourceCat):
375 Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
376 if pixToTanPix:
377 p = lsst.geom.Point2D(source.getX(), source.getY())
378 linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear()
379 m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
380 m.transform(linTransform)
381 Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
382
383 xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
384
385 width = numpy.sqrt(0.5*(xx + yy))
386 with numpy.errstate(invalid="ignore"): # suppress NAN warnings
387 bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
388 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
389 bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
390 if self.config.doFluxLimit:
391 bad = numpy.logical_or(bad, flux < self.config.fluxMin)
392 if self.config.fluxMax > 0:
393 bad = numpy.logical_or(bad, flux > self.config.fluxMax)
394 if self.config.doSignalToNoiseLimit:
395 bad = numpy.logical_or(bad, flux/fluxErr < self.config.signalToNoiseMin)
396 if self.config.signalToNoiseMax > 0:
397 bad = numpy.logical_or(bad, flux/fluxErr > self.config.signalToNoiseMax)
398 bad = numpy.logical_or(bad, width < self.config.widthMin)
399 bad = numpy.logical_or(bad, width > self.config.widthMax)
400 good = numpy.logical_not(bad)
401
402 if not numpy.any(good):
403 raise RuntimeError("No objects passed our cuts for consideration as psf stars")
404
405 mag = -2.5*numpy.log10(flux[good])
406 width = width[good]
407 #
408 # Look for the maximum in the size histogram, then search upwards for the minimum that separates
409 # the initial peak (of, we presume, stars) from the galaxies
410 #
411 if dumpData:
412 import os
413 import pickle as pickle
414 _ii = 0
415 while True:
416 pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
417 if not os.path.exists(pickleFile):
418 break
419 _ii += 1
420
421 with open(pickleFile, "wb") as fd:
422 pickle.dump(mag, fd, -1)
423 pickle.dump(width, fd, -1)
424
425 centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
426 widthStdAllowed=self.config.widthStdAllowed)
427
428 if display and plotMagSize:
429 fig = plot(mag, width, centers, clusterId,
430 magType=self.config.sourceFluxField.split(".")[-1].title(),
431 marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
432 else:
433 fig = None
434
435 clusterId = _improveCluster(width, centers, clusterId,
436 nsigma=self.config.nSigmaClip,
437 widthStdAllowed=self.config.widthStdAllowed)
438
439 if display and plotMagSize:
440 plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
441
442 stellar = (clusterId == 0)
443 #
444 # We know enough to plot, if so requested
445 #
446 frame = 0
447
448 if fig:
449 if display and displayExposure:
450 disp = afwDisplay.Display(frame=frame)
451 disp.mtv(exposure.getMaskedImage(), title="PSF candidates")
452
453 global eventHandler
454 eventHandler = EventHandler(fig.get_axes()[0], mag, width,
455 sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame])
456
457 fig.show()
458
459 while True:
460 try:
461 reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
462 except EOFError:
463 reply = None
464 if not reply:
465 reply = "c"
466
467 if reply:
468 if reply[0] == "h":
469 print("""\
470 We cluster the points; red are the stellar candidates and the other colours are other clusters.
471 Points labelled + are rejects from the cluster (only for cluster 0).
472
473 At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
474
475 If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the
476 image display.
477 """)
478 elif reply[0] == "p":
479 import pdb
480 pdb.set_trace()
481 elif reply[0] == 'q':
482 sys.exit(1)
483 else:
484 break
485
486 if display and displayExposure:
487 mi = exposure.getMaskedImage()
488 with disp.Buffering():
489 for i, source in enumerate(sourceCat):
490 if good[i]:
491 ctype = afwDisplay.GREEN # star candidate
492 else:
493 ctype = afwDisplay.RED # not star
494
495 disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype)
496
497 # stellar only applies to good==True objects
498 mask = good == True # noqa (numpy bool comparison): E712
499 good[mask] = stellar
500
501 return Struct(selected=good)
def plot(mag, width, centers, clusterId, marker="o", markersize=2, markeredgewidth=0, ltype='-', magType="model", clear=True)