lsst.meas.extensions.astrometryNet  master-gb035c22750+31
multiindex.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 __all__ = ["getIndexPath", "getConfigFromEnvironment", "AstrometryNetCatalog", "generateCache"]
4 
5 from builtins import zip
6 from builtins import range
7 from builtins import object
8 import os
9 
10 import numpy as np
11 import pyfits
12 
13 import lsst.utils
14 from lsst.log import Log
15 from .astrometry_net import MultiIndex, healpixDistance
16 from .astrometryNetDataConfig import AstrometryNetDataConfig
17 
18 
19 def getIndexPath(fn):
20  """!Get the path to the specified astrometry.net index file
21 
22  No effort is made to confirm that the file exists, so it may be used to locate the
23  path to a non-existent file (e.g., to write).
24 
25  @param[in] fn path to index file; if relative, then relative to astrometry_net_data
26  if that product is setup, else relative to the current working directory
27  @return the absolute path to the index file
28  """
29  if os.path.isabs(fn):
30  return fn
31  try:
32  andir = lsst.utils.getPackageDir('astrometry_net_data')
33  except:
34  # Relative to cwd
35  return os.path.abspath(fn)
36  return os.path.join(andir, fn)
37 
38 
40  """Find the config file from the environment
41 
42  The andConfig.py file is in the astrometry_net_data directory.
43  """
44  try:
45  anDir = lsst.utils.getPackageDir('astrometry_net_data')
46  except:
47  anDir = os.getcwd()
48  andConfigPath = "andConfig.py"
49  if not os.path.exists(andConfigPath):
50  raise RuntimeError("Unable to find andConfig.py in the current directory. "
51  "Did you forget to setup astrometry_net_data?")
52  else:
53  andConfigPath = os.path.join(anDir, "andConfig.py")
54  if not os.path.exists(andConfigPath):
55  raise RuntimeError("Unable to find andConfig.py in astrometry_net_data directory %s" % (anDir,))
56 
57  andConfig = AstrometryNetDataConfig()
58  andConfig.load(andConfigPath)
59  return andConfig
60 
61 
62 class MultiIndexCache(object):
63  """A wrapper for the multiindex_t, which only reads the data when it needs to
64 
65  The MultiIndexCache may be instantiated directly, or via the 'fromFilenameList'
66  class method, which loads it from a list of filenames.
67  """
68 
69  def __init__(self, filenameList, healpix, nside):
70  """!Constructor
71 
72  @param filenameList List of filenames; first is the multiindex, then
73  follows the individual index files
74  @param healpix Healpix number
75  @param nside Healpix nside
76  """
77  if len(filenameList) < 2:
78  raise RuntimeError("Insufficient filenames provided for multiindex (%s): expected >= 2" %
79  (filenameList,))
80  self._filenameList = filenameList
81  self._healpix = int(healpix)
82  self._nside = int(nside)
83  self._mi = None
84  self._loaded = False
85  self.log = Log.getDefaultLogger()
86 
87  @classmethod
88  def fromFilenameList(cls, filenameList):
89  """Construct from a list of filenames
90 
91  The list of filenames should contain the multiindex filename first,
92  then the individual index filenames. The healpix and nside are
93  determined by reading the indices, so this is not very efficient.
94  """
95  self = cls(filenameList, 0, 0)
96  self.reload()
97  healpixes = set(self[i].healpix for i in range(len(self)))
98  nsides = set(self[i].hpnside for i in range(len(self)))
99  assert len(healpixes) == 1
100  assert len(nsides) == 1
101  self._healpix = healpixes.pop()
102  self._nside = nsides.pop()
103  return self
104 
105  def read(self):
106  """Read the indices"""
107  if self._mi is not None:
108  return
109  fn = getIndexPath(self._filenameList[0])
110  if not os.path.exists(fn):
111  raise RuntimeError(
112  "Unable to get filename for astrometry star file %s" % (self._filenameList[0],))
113  self._mi = MultiIndex(fn)
114  if self._mi is None:
115  # Can't proceed at all without stars
116  raise RuntimeError('Failed to read stars from astrometry multiindex filename "%s"' % fn)
117  for i, fn in enumerate(self._filenameList[1:]):
118  if fn is None:
119  self.log.debug('Unable to find index part of multiindex %s', fn)
120  continue
121  fn = getIndexPath(fn)
122  if not os.path.exists(fn):
123  self.log.warn("Unable to get filename for astrometry index %s", fn)
124  continue
125  self.log.debug('Reading index from multiindex file "%s"', fn)
126  self._mi.addIndex(fn, False)
127  ind = self._mi[i]
128  self.log.debug(' index %i, hp %i (nside %i), nstars %i, nquads %i',
129  ind.indexid, ind.healpix, ind.hpnside, ind.nstars, ind.nquads)
130 
131  def reload(self):
132  """Reload the indices."""
133  if self._loaded:
134  return
135  if self._mi is None:
136  self.read()
137  else:
138  self._mi.reload()
139  self._loaded = True
140 
141  def unload(self):
142  """Unload the indices"""
143  if not self._loaded:
144  return
145  self._mi.unload()
146  self._loaded = False
147 
148  def isWithinRange(self, coord, distance):
149  """!Is the index within range of the provided coordinates?
150 
151  @param coord Coordinate to check (lsst.afw.coord.Coord)
152  @param distance Angular distance (lsst.afw.geom.Angle)
153  """
154  return (self._healpix == -1 or healpixDistance(self._healpix, self._nside, coord) <= distance)
155 
156  def __getitem__(self, i):
157  self.reload()
158  return self._mi[i]
159 
160  def __len__(self):
161  return len(self._filenameList) - 1 # The first is the multiindex; the rest are the indices
162 
163  def __iter__(self):
164  self.reload()
165  return iter(self._mi)
166 
167 
168 class AstrometryNetCatalog(object):
169  """An interface to an astrometry.net catalog
170 
171  Behaves like a list of MultiIndexCache (or multiindex_t).
172 
173  These should usually be constructed using the 'fromEnvironment'
174  class method, which wraps the 'fromIndexFiles' and 'fromCache'
175  alternative class methods.
176  """
177  _cacheFilename = "andCache.fits"
178 
179  def __init__(self, andConfig):
180  """!Constructor
181 
182  @param andConfig Configuration (an AstrometryNetDataConfig)
183  """
184  self.config = andConfig
185  cacheName = getIndexPath(self._cacheFilename)
186  if self.config.allowCache and os.path.exists(cacheName):
187  self._initFromCache(cacheName)
188  else:
189  self._initFromIndexFiles(self.config)
190 
191  def _initFromIndexFiles(self, andConfig):
192  """Initialise from the index files in an AstrometryNetDataConfig"""
193  indexFiles = list(zip(andConfig.indexFiles, andConfig.indexFiles)) + andConfig.multiIndexFiles
194  self._multiInds = [MultiIndexCache.fromFilenameList(fnList) for fnList in indexFiles]
195 
196  def writeCache(self):
197  """Write a cache file
198 
199  The cache file is a FITS file with all the required information to build the
200  AstrometryNetCatalog quickly. The first table extension contains a row for each multiindex,
201  storing the healpix and nside values. The second table extension contains a row
202  for each filename in all the multiindexes. The two may be JOINed through the
203  'id' column.
204  """
205  outName = getIndexPath(self._cacheFilename)
206  numFilenames = sum(len(ind._filenameList) for ind in self._multiInds)
207  maxLength = max(len(fn) for ind in self._multiInds for fn in ind._filenameList) + 1
208 
209  # First table
210  first = pyfits.new_table([pyfits.Column(name="id", format="K"),
211  pyfits.Column(name="healpix", format="K"),
212  pyfits.Column(name="nside", format="K"),
213  ], nrows=len(self._multiInds))
214  first.data.field("id")[:] = np.arange(len(self._multiInds), dtype=int)
215  first.data.field("healpix")[:] = np.array([ind._healpix for ind in self._multiInds])
216  first.data.field("nside")[:] = np.array([ind._nside for ind in self._multiInds])
217 
218  # Second table
219  second = pyfits.new_table([pyfits.Column(name="id", format="K"),
220  pyfits.Column(name="filename", format="%dA" % (maxLength)),
221  ], nrows=numFilenames)
222  ident = second.data.field("id")
223  filenames = second.data.field("filename")
224  i = 0
225  for j, ind in enumerate(self._multiInds):
226  for fn in ind._filenameList:
227  ident[i] = j
228  filenames[i] = fn
229  i += 1
230 
231  pyfits.HDUList([pyfits.PrimaryHDU(), first, second]).writeto(outName, clobber=True)
232 
233  def _initFromCache(self, filename):
234  """Initialise from a cache file
235 
236  Ingest the cache file written by the 'writeCache' method and
237  use that to quickly instantiate the AstrometryNetCatalog.
238  """
239  with pyfits.open(filename) as hduList:
240  first = hduList[1].data
241  second = hduList[2].data
242 
243  # first JOIN second USING(id)
244  filenames = {i: [] for i in first.field("id")}
245  for id2, fn in zip(second.field("id"), second.field("filename")):
246  filenames[id2].append(fn)
247  self._multiInds = [MultiIndexCache(filenames[i], hp, nside) for i, hp, nside in
248  zip(first.field("id"), first.field("healpix"), first.field("nside"))]
249 
250  # Check for consistency
251  cacheFiles = set(second.field("filename"))
252  configFiles = set(sum(self.config.multiIndexFiles, []) + self.config.indexFiles)
253  assert(cacheFiles == configFiles)
254 
255  def __getitem__(self, ii):
256  return self._multiInds[ii]
257 
258  def __iter__(self):
259  return iter(self._multiInds)
260 
261  def __len__(self):
262  return len(self._multiInds)
263 
264 
265 def generateCache(andConfig=None):
266  """Generate a cache file"""
267  if andConfig is None:
268  andConfig = getConfigFromEnvironment()
269  catalog = AstrometryNetCatalog(andConfig)
270  try:
271  for index in catalog:
272  index.reload()
273  catalog.writeCache()
274  finally:
275  for index in catalog:
276  index.unload()
def isWithinRange(self, coord, distance)
Is the index within range of the provided coordinates?
Definition: multiindex.py:148
def getIndexPath(fn)
Get the path to the specified astrometry.net index file.
Definition: multiindex.py:19
def __init__(self, filenameList, healpix, nside)
Constructor.
Definition: multiindex.py:69
lsst::afw::geom::Angle healpixDistance(int hp, int nside, lsst::afw::coord::Coord const &coord)
Calculate the distance from coordinates to a healpix.