lsst.meas.algorithms  13.0-16-g6e7f056
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
secondMomentStarSelector.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from builtins import range
24 from builtins import object
25 import collections
26 import math
27 
28 import numpy
29 
30 from lsst.afw.cameraGeom import TAN_PIXELS
31 from lsst.afw.geom.ellipses import Quadrupole
32 from lsst.afw.table import SourceCatalog, SourceTable
33 from lsst.pipe.base import Struct
34 import lsst.pex.config as pexConfig
35 import lsst.afw.detection as afwDetection
36 import lsst.afw.display.ds9 as ds9
37 import lsst.afw.image as afwImage
38 import lsst.afw.math as afwMath
39 import lsst.afw.geom as afwGeom
40 from .psfCandidate import makePsfCandidate
41 from .doubleGaussianPsf import DoubleGaussianPsf
42 from lsst.meas.base import SingleFrameMeasurementTask, SingleFrameMeasurementConfig
43 from .starSelector import BaseStarSelectorTask, starSelectorRegistry
44 
45 
46 class SecondMomentStarSelectorConfig(BaseStarSelectorTask.ConfigClass):
47  fluxLim = pexConfig.Field(
48  doc="specify the minimum psfFlux for good Psf Candidates",
49  dtype=float,
50  default=12500.0,
51  check=lambda x: x >= 0.0,
52  )
53  fluxMax = pexConfig.Field(
54  doc="specify the maximum psfFlux for good Psf Candidates (ignored if == 0)",
55  dtype=float,
56  default=0.0,
57  check=lambda x: x >= 0.0,
58  )
59  clumpNSigma = pexConfig.Field(
60  doc="candidate PSF's shapes must lie within this many sigma of the average shape",
61  dtype=float,
62  default=2.0,
63  check=lambda x: x >= 0.0,
64  )
65  histSize = pexConfig.Field(
66  doc="Number of bins in moment histogram",
67  dtype=int,
68  default=64,
69  check=lambda x: x > 0,
70  )
71  histMomentMax = pexConfig.Field(
72  doc="Maximum moment to consider",
73  dtype=float,
74  default=100.0,
75  check=lambda x: x > 0,
76  )
77  histMomentMaxMultiplier = pexConfig.Field(
78  doc="Multiplier of mean for maximum moments histogram range",
79  dtype=float,
80  default=5.0,
81  check=lambda x: x > 0,
82  )
83  histMomentClip = pexConfig.Field(
84  doc="Clipping threshold for moments histogram range",
85  dtype=float,
86  default=5.0,
87  check=lambda x: x > 0,
88  )
89  histMomentMinMultiplier = pexConfig.Field(
90  doc="Multiplier of mean for minimum moments histogram range",
91  dtype=float,
92  default=2.0,
93  check=lambda x: x > 0,
94  )
95 
96  def setDefaults(self):
97  BaseStarSelectorTask.ConfigClass.setDefaults(self)
98  self.badFlags = [
99  "base_PixelFlags_flag_edge",
100  "base_PixelFlags_flag_interpolatedCenter",
101  "base_PixelFlags_flag_saturatedCenter",
102  "base_PixelFlags_flag_crCenter",
103  ]
104 
105 Clump = collections.namedtuple('Clump', ['peak', 'x', 'y', 'ixx', 'ixy', 'iyy', 'a', 'b', 'c'])
106 
107 
108 class CheckSource(object):
109 
110  """A functor to check whether a source has any flags set that should cause it to be labeled bad."""
111 
112  def __init__(self, table, badFlags, fluxLim, fluxMax):
113  self.keys = [table.getSchema().find(name).key for name in badFlags]
114  self.keys.append(table.getCentroidFlagKey())
115  self.fluxLim = fluxLim
116  self.fluxMax = fluxMax
117 
118  def __call__(self, source):
119  for k in self.keys:
120  if source.get(k):
121  return False
122  if self.fluxLim is not None and source.getPsfFlux() < self.fluxLim: # ignore faint objects
123  return False
124  if self.fluxMax != 0.0 and source.getPsfFlux() > self.fluxMax: # ignore bright objects
125  return False
126  return True
127 
128 ## \addtogroup LSST_task_documentation
129 ## \{
130 ## \page SecondMomentStarSelectorTask
131 ## \ref SecondMomentStarSelectorTask_ "SecondMomentStarSelectorTask"
132 ## \copybrief SecondMomentStarSelectorTask
133 ## \}
134 
135 
136 class SecondMomentStarSelectorTask(BaseStarSelectorTask):
137  """!A star selector based on second moments
138 
139  @anchor SecondMomentStarSelectorTask_
140 
141  @section meas_algorithms_secondMomentStarSelector_Contents Contents
142 
143  - @ref meas_algorithms_secondMomentStarSelector_Purpose
144  - @ref meas_algorithms_secondMomentStarSelector_Initialize
145  - @ref meas_algorithms_secondMomentStarSelector_IO
146  - @ref meas_algorithms_secondMomentStarSelector_Config
147  - @ref meas_algorithms_secondMomentStarSelector_Debug
148 
149  @section meas_algorithms_secondMomentStarSelector_Purpose Description
150 
151  A star selector based on second moments.
152 
153  @warning This is a naive algorithm; use with caution.
154 
155  @section meas_algorithms_secondMomentStarSelector_Initialize Task initialisation
156 
157  @copydoc \_\_init\_\_
158 
159  @section meas_algorithms_secondMomentStarSelector_IO Invoking the Task
160 
161  Like all star selectors, the main method is `run`.
162 
163  @section meas_algorithms_secondMomentStarSelector_Config Configuration parameters
164 
165  See @ref SecondMomentStarSelectorConfig
166 
167  @section meas_algorithms_secondMomentStarSelector_Debug Debug variables
168 
169  SecondMomentStarSelectorTask has a debug dictionary with the following keys:
170  <dl>
171  <dt>display
172  <dd>bool; if True display debug information
173  </dl>
174  display = lsstDebug.Info(__name__).display
175  displayExposure = lsstDebug.Info(__name__).displayExposure
176  pauseAtEnd = lsstDebug.Info(__name__).pauseAtEnd
177 
178  For example, put something like:
179  @code{.py}
180  import lsstDebug
181  def DebugInfo(name):
182  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
183  if name.endswith("catalogStarSelector"):
184  di.display = True
185 
186  return di
187 
188  lsstDebug.Info = DebugInfo
189  @endcode
190  into your `debug.py` file and run your task with the `--debug` flag.
191  """
192  ConfigClass = SecondMomentStarSelectorConfig
193  usesMatches = False # selectStars does not use its matches argument
194 
195  def selectStars(self, exposure, sourceCat, matches=None):
196  """!Return a list of PSF candidates that represent likely stars
197 
198  A list of PSF candidates may be used by a PSF fitter to construct a PSF.
199 
200  @param[in] exposure the exposure containing the sources
201  @param[in] sourceCat catalog of sources that may be stars (an lsst.afw.table.SourceCatalog)
202  @param[in] matches astrometric matches; ignored by this star selector
203 
204  @return an lsst.pipe.base.Struct containing:
205  - starCat catalog of selected stars (a subset of sourceCat)
206  """
207  import lsstDebug
208  display = lsstDebug.Info(__name__).display
209 
210  isGoodSource = CheckSource(sourceCat.getTable(), self.config.badFlags, self.config.fluxLim,
211  self.config.fluxMax)
212 
213  detector = exposure.getDetector()
214 
215  mi = exposure.getMaskedImage()
216  #
217  # Create an Image of Ixx v. Iyy, i.e. a 2-D histogram
218  #
219 
220  # Use stats on our Ixx/yy values to determine the xMax/yMax range for clump image
221  iqqList = []
222  for s in sourceCat:
223  ixx, iyy = s.getIxx(), s.getIyy()
224  # ignore NaN and unrealistically large values
225  if (ixx == ixx and ixx < self.config.histMomentMax and
226  iyy == iyy and iyy < self.config.histMomentMax and
227  isGoodSource(s)):
228  iqqList.append(s.getIxx())
229  iqqList.append(s.getIyy())
230  stat = afwMath.makeStatistics(iqqList, afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.MAX)
231  iqqMean = stat.getValue(afwMath.MEANCLIP)
232  iqqStd = stat.getValue(afwMath.STDEVCLIP)
233  iqqMax = stat.getValue(afwMath.MAX)
234 
235  iqqLimit = max(iqqMean + self.config.histMomentClip*iqqStd,
236  self.config.histMomentMaxMultiplier*iqqMean)
237  # if the max value is smaller than our range, use max as the limit, but don't go below N*mean
238  if iqqLimit > iqqMax:
239  iqqLimit = max(self.config.histMomentMinMultiplier*iqqMean, iqqMax)
240 
241  psfHist = _PsfShapeHistogram(detector=detector,
242  xSize=self.config.histSize, ySize=self.config.histSize,
243  ixxMax=iqqLimit, iyyMax=iqqLimit)
244 
245  if display:
246  frame = 0
247  ds9.mtv(mi, frame=frame, title="PSF candidates")
248  ctypes = []
249 
250  for source in sourceCat:
251  good = isGoodSource(source)
252  if good:
253  notRejected = psfHist.insert(source)
254  if display:
255  if good:
256  if notRejected:
257  ctypes.append(ds9.GREEN) # good
258  else:
259  ctypes.append(ds9.MAGENTA) # rejected
260  else:
261  ctypes.append(ds9.RED) # bad
262 
263  if display:
264  with ds9.Buffering():
265  for source, ctype in zip(sourceCat, ctypes):
266  ds9.dot("o", source.getX() - mi.getX0(), source.getY() - mi.getY0(),
267  frame=frame, ctype=ctype)
268 
269  clumps = psfHist.getClumps(display=display)
270 
271  #
272  # Go through and find all the PSF-like objects
273  #
274  # We'll split the image into a number of cells, each of which contributes only
275  # one PSF candidate star
276  #
277  starCat = SourceCatalog(sourceCat.table)
278 
279  pixToTanXYTransform = None
280  if detector is not None:
281  tanSys = detector.makeCameraSys(TAN_PIXELS)
282  pixToTanXYTransform = detector.getTransformMap().get(tanSys)
283 
284  # psf candidate shapes must lie within this many RMS of the average shape
285  # N.b. if Ixx == Iyy, Ixy = 0 the criterion is
286  # dx^2 + dy^2 < self.config.clumpNSigma*(Ixx + Iyy) == 2*self.config.clumpNSigma*Ixx
287  for source in sourceCat:
288  if not isGoodSource(source):
289  continue
290  Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
291  if pixToTanXYTransform:
292  p = afwGeom.Point2D(source.getX(), source.getY())
293  linTransform = pixToTanXYTransform.linearizeForwardTransform(p).getLinear()
294  m = Quadrupole(Ixx, Iyy, Ixy)
295  m.transform(linTransform)
296  Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
297 
298  x, y = psfHist.momentsToPixel(Ixx, Iyy)
299  for clump in clumps:
300  dx, dy = (x - clump.x), (y - clump.y)
301 
302  if math.sqrt(clump.a*dx*dx + 2*clump.b*dx*dy + clump.c*dy*dy) < 2*self.config.clumpNSigma:
303  # A test for > would be confused by NaN
304  if not isGoodSource(source):
305  continue
306  try:
307  psfCandidate = makePsfCandidate(source, exposure)
308 
309  # The setXXX methods are class static, but it's convenient to call them on
310  # an instance as we don't know Exposure's pixel type
311  # (and hence psfCandidate's exact type)
312  if psfCandidate.getWidth() == 0:
313  psfCandidate.setBorderWidth(self.config.borderWidth)
314  psfCandidate.setWidth(self.config.kernelSize + 2*self.config.borderWidth)
315  psfCandidate.setHeight(self.config.kernelSize + 2*self.config.borderWidth)
316 
317  im = psfCandidate.getMaskedImage().getImage()
318  if not numpy.isfinite(afwMath.makeStatistics(im, afwMath.MAX).getValue()):
319  continue
320  starCat.append(source)
321 
322  if display:
323  ds9.dot("o", source.getX() - mi.getX0(), source.getY() - mi.getY0(),
324  size=4, frame=frame, ctype=ds9.CYAN)
325  except Exception as err:
326  self.log.error("Failed on source %s: %s" % (source.getId(), err))
327  break
328 
329  return Struct(
330  starCat=starCat,
331  )
332 
333 
334 class _PsfShapeHistogram(object):
335 
336  """A class to represent a histogram of (Ixx, Iyy)
337  """
338 
339  def __init__(self, xSize=32, ySize=32, ixxMax=30, iyyMax=30, detector=None, xy0=afwGeom.Point2D(0, 0)):
340  """Construct a _PsfShapeHistogram
341 
342  The maximum seeing FWHM that can be tolerated is [xy]Max/2.35 pixels.
343  The 'resolution' of stars vs galaxies/CRs is provided by [xy]Size/[xy]Max.
344  A larger (better) resolution may thresh the peaks, but a smaller (worse)
345  resolution will allow stars and galaxies/CRs to mix. The disadvantages of
346  a larger (better) resolution can be compensated (some) by using multiple
347  histogram peaks.
348 
349  @input[in] [xy]Size: the size of the psfImage (in pixels)
350  @input[in] ixxMax, iyyMax: the maximum values for I[xy][xy]
351  """
352  self._xSize, self._ySize = xSize, ySize
353  self._xMax, self._yMax = ixxMax, iyyMax
354  self._psfImage = afwImage.ImageF(afwGeom.ExtentI(xSize, ySize), 0)
355  self._num = 0
356  self.detector = detector
357  self.xy0 = xy0
358 
359  def getImage(self):
360  return self._psfImage
361 
362  def insert(self, source):
363  """Insert source into the histogram."""
364 
365  ixx, iyy, ixy = source.getIxx(), source.getIyy(), source.getIxy()
366  if self.detector:
367  tanSys = self.detector.makeCameraSys(TAN_PIXELS)
368  if tanSys in self.detector.getTransformMap():
369  pixToTanXYTransform = self.detector.getTransformMap()[tanSys]
370  p = afwGeom.Point2D(source.getX(), source.getY())
371  linTransform = pixToTanXYTransform.linearizeForwardTransform(p).getLinear()
372  m = Quadrupole(ixx, iyy, ixy)
373  m.transform(linTransform)
374  ixx, iyy, ixy = m.getIxx(), m.getIyy(), m.getIxy()
375 
376  try:
377  pixel = self.momentsToPixel(ixx, iyy)
378  i = int(pixel[0])
379  j = int(pixel[1])
380  except:
381  return 0
382 
383  if i in range(0, self._xSize) and j in range(0, self._ySize):
384  if i != 0 or j != 0:
385  self._psfImage.set(i, j, self._psfImage.get(i, j) + 1)
386  self._num += 1
387  return 1 # success
388 
389  return 0 # failure
390 
391  def momentsToPixel(self, ixx, iyy):
392  #x = math.sqrt(ixx) * self._xSize / self._xMax
393  #y = math.sqrt(iyy) * self._ySize / self._yMax
394  x = ixx * self._xSize / self._xMax
395  y = iyy * self._ySize / self._yMax
396  return x, y
397 
398  def pixelToMoments(self, x, y):
399  """Given a peak position in self._psfImage, return the corresponding (Ixx, Iyy)"""
400 
401  #ixx = (x*self._xMax/self._xSize)**2
402  #iyy = (y*self._yMax/self._ySize)**2
403  ixx = x*self._xMax/self._xSize
404  iyy = y*self._yMax/self._ySize
405  return ixx, iyy
406 
407  def getClumps(self, sigma=1.0, display=False):
408  if self._num <= 0:
409  raise RuntimeError("No candidate PSF sources")
410 
411  psfImage = self.getImage()
412  #
413  # Embed psfImage into a larger image so we can smooth when measuring it
414  #
415  width, height = psfImage.getWidth(), psfImage.getHeight()
416  largeImg = psfImage.Factory(afwGeom.ExtentI(2*width, 2*height))
417  largeImg.set(0)
418 
419  bbox = afwGeom.BoxI(afwGeom.PointI(width, height), afwGeom.ExtentI(width, height))
420  largeImg.assign(psfImage, bbox, afwImage.LOCAL)
421  #
422  # Now measure that image, looking for the highest peak. Start by building an Exposure
423  #
424  msk = afwImage.Mask(largeImg.getDimensions())
425  msk.set(0)
426  var = afwImage.ImageF(largeImg.getDimensions())
427  var.set(1)
428  mpsfImage = afwImage.MaskedImageF(largeImg, msk, var)
429  mpsfImage.setXY0(afwGeom.PointI(-width, -height))
430  del msk
431  del var
432  exposure = afwImage.makeExposure(mpsfImage)
433 
434  #
435  # Next run an object detector
436  #
437  maxVal = afwMath.makeStatistics(psfImage, afwMath.MAX).getValue()
438  threshold = maxVal - sigma*math.sqrt(maxVal)
439  if threshold <= 0.0:
440  threshold = maxVal
441 
442  threshold = afwDetection.Threshold(threshold)
443 
444  ds = afwDetection.FootprintSet(mpsfImage, threshold, "DETECTED")
445  #
446  # And measure it. This policy isn't the one we use to measure
447  # Sources, it's only used to characterize this PSF histogram
448  #
449  schema = SourceTable.makeMinimalSchema()
450  psfImageConfig = SingleFrameMeasurementConfig()
451  psfImageConfig.slots.centroid = "base_SdssCentroid"
452  psfImageConfig.plugins["base_SdssCentroid"].doFootprintCheck = False
453  psfImageConfig.slots.psfFlux = None # "base_PsfFlux"
454  psfImageConfig.slots.apFlux = "base_CircularApertureFlux_3_0"
455  psfImageConfig.slots.modelFlux = None
456  psfImageConfig.slots.instFlux = None
457  psfImageConfig.slots.calibFlux = None
458  psfImageConfig.slots.shape = "base_SdssShape"
459  # Formerly, this code had centroid.sdss, flux.psf, flux.naive,
460  # flags.pixel, and shape.sdss
461  psfImageConfig.algorithms.names = ["base_SdssCentroid", "base_CircularApertureFlux", "base_SdssShape"]
462  psfImageConfig.algorithms["base_CircularApertureFlux"].radii = [3.0]
463  psfImageConfig.validate()
464  task = SingleFrameMeasurementTask(schema, config=psfImageConfig)
465 
466  sourceCat = SourceCatalog(schema)
467 
468  gaussianWidth = 1.5 # Gaussian sigma for detection convolution
469  exposure.setPsf(DoubleGaussianPsf(11, 11, gaussianWidth))
470 
471  ds.makeSources(sourceCat)
472  #
473  # Show us the Histogram
474  #
475  if display:
476  frame = 1
477  dispImage = mpsfImage.Factory(mpsfImage, afwGeom.BoxI(afwGeom.PointI(width, height),
478  afwGeom.ExtentI(width, height)),
479  afwImage.LOCAL)
480  ds9.mtv(dispImage, title="PSF Selection Image", frame=frame)
481 
482  clumps = list() # List of clumps, to return
483  e = None # thrown exception
484  IzzMin = 1.0 # Minimum value for second moments
485  IzzMax = (self._xSize/8.0)**2 # Max value ... clump radius should be < clumpImgSize/8
486  apFluxes = []
487  task.run(sourceCat, exposure) # notes that this is backwards for the new framework
488  for i, source in enumerate(sourceCat):
489  if source.getCentroidFlag():
490  continue
491  x, y = source.getX(), source.getY()
492 
493  apFluxes.append(source.getApFlux())
494 
495  val = mpsfImage.getImage().get(int(x) + width, int(y) + height)
496 
497  psfClumpIxx = source.getIxx()
498  psfClumpIxy = source.getIxy()
499  psfClumpIyy = source.getIyy()
500 
501  if display:
502  if i == 0:
503  ds9.pan(x, y, frame=frame)
504 
505  ds9.dot("+", x, y, ctype=ds9.YELLOW, frame=frame)
506  ds9.dot("@:%g,%g,%g" % (psfClumpIxx, psfClumpIxy, psfClumpIyy), x, y,
507  ctype=ds9.YELLOW, frame=frame)
508 
509  if psfClumpIxx < IzzMin or psfClumpIyy < IzzMin:
510  psfClumpIxx = max(psfClumpIxx, IzzMin)
511  psfClumpIyy = max(psfClumpIyy, IzzMin)
512  if display:
513  ds9.dot("@:%g,%g,%g" % (psfClumpIxx, psfClumpIxy, psfClumpIyy), x, y,
514  ctype=ds9.RED, frame=frame)
515 
516  det = psfClumpIxx*psfClumpIyy - psfClumpIxy*psfClumpIxy
517  try:
518  a, b, c = psfClumpIyy/det, -psfClumpIxy/det, psfClumpIxx/det
519  except ZeroDivisionError:
520  a, b, c = 1e4, 0, 1e4
521 
522  clumps.append(Clump(peak=val, x=x, y=y, a=a, b=b, c=c,
523  ixx=psfClumpIxx, ixy=psfClumpIxy, iyy=psfClumpIyy))
524 
525  if len(clumps) == 0:
526  msg = "Failed to determine center of PSF clump"
527  if e:
528  msg += ": %s" % e
529  raise RuntimeError(msg)
530 
531  # if it's all we got return it
532  if len(clumps) == 1:
533  return clumps
534 
535  # which clump is the best?
536  # if we've undistorted the moments, stars should only have 1 clump
537  # use the apFlux from the clump measurement, and take the highest
538  # ... this clump has more psf star candidate neighbours than the others.
539 
540  # get rid of any that are huge, and thus poorly defined
541  goodClumps = []
542  for clump in clumps:
543  if clump.ixx < IzzMax and clump.iyy < IzzMax:
544  goodClumps.append(clump)
545 
546  # if culling > IzzMax cost us all clumps, we'll have to take what we have
547  if len(goodClumps) == 0:
548  goodClumps = clumps
549 
550  # use the 'brightest' clump
551  iBestClump = numpy.argsort(apFluxes)[0]
552  clumps = [clumps[iBestClump]]
553  return clumps
554 
555 starSelectorRegistry.register("secondMoment", SecondMomentStarSelectorTask)
def selectStars
Return a list of PSF candidates that represent likely stars.
std::shared_ptr< PsfCandidate< PixelT > > makePsfCandidate(boost::shared_ptr< afw::table::SourceRecord > const &source, boost::shared_ptr< afw::image::Exposure< PixelT > > image)
Return a PsfCandidate of the right sort.
Definition: PsfCandidate.h:182