lsst.base  16.0-1-g928b041
LSST Data Management Base Package
packages.py
Go to the documentation of this file.
1 # This file is part of base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """
22 Determine which packages are being used in the system and their versions
23 
24 There are a few different types of packages, and their versions are collected in different ways:
25 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from interrogating the dynamic library
26 2. Python modules (e.g., afw, numpy; galsim is also in this group even though we only use it through the
27  library, because no version information is currently provided through the library): we get their version
28  from the __version__ module variable. Note that this means that we're only aware of modules that have
29  already been imported.
30 3. Other packages provide no run-time accessible version information (e.g., astrometry_net): we get their
31  version from interrogating the environment. Currently, that means EUPS; if EUPS is replaced or dropped then
32  we'll need to consider an alternative means of getting this version information.
33 4. Local versions of packages (a non-installed EUPS package, selected with "setup -r /path/to/package"): we
34  identify these through the environment (EUPS again) and use as a version the path supplemented with the
35  git SHA and, if the git repo isn't clean, an MD5 of the diff.
36 
37 These package versions are collected and stored in a Packages object, which provides useful comparison and
38 persistence features.
39 
40 Example usage:
41 
42  from lsst.base import Packages
43  pkgs = Packages.fromSystem()
44  print "Current packages:", pkgs
45  old = Packages.read("/path/to/packages.pickle")
46  print "Old packages:", old
47  print "Missing packages compared to before:", pkgs.missing(old)
48  print "Extra packages compared to before:", pkgs.extra(old)
49  print "Different packages: ", pkgs.difference(old)
50  old.update(pkgs) # Include any new packages in the old
51  old.write("/path/to/packages.pickle")
52 """
53 from builtins import object
54 
55 import os
56 import sys
57 import hashlib
58 import importlib
59 import subprocess
60 import pickle as pickle
61 from collections import Mapping
62 
63 from .versions import getRuntimeVersions
64 
65 from future import standard_library
66 standard_library.install_aliases()
67 
68 __all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
69 
70 
71 # Packages used at build-time (e.g., header-only)
72 BUILDTIME = set(["boost", "eigen", "tmv"])
73 
74 # Python modules to attempt to load so we can try to get the version
75 # We do this because the version only appears to be available from python, but we use the library
76 PYTHON = set(["galsim"])
77 
78 # Packages that don't seem to have a mechanism for reporting the runtime version
79 # We need to guess the version from the environment
80 ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
81 
82 
84  """Determine the version of a python module.
85 
86  Parameters
87  ----------
88  module : `module`
89  Module for which to get version.
90 
91  Returns
92  -------
93  version : `str`
94 
95  Raises
96  ------
97  AttributeError
98  Raised if __version__ attribute is not set.
99 
100  Notes
101  -----
102  We supplement the version with information from the __dependency_versions__
103  (a specific variable set by LSST's sconsUtils at build time) only for packages
104  that are typically used only at build-time.
105  """
106  version = module.__version__
107  if hasattr(module, "__dependency_versions__"):
108  # Add build-time dependencies
109  deps = module.__dependency_versions__
110  buildtime = BUILDTIME & set(deps.keys())
111  if buildtime:
112  version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
113  for pkg in sorted(buildtime))
114  return version
115 
116 
118  """Get imported python packages and their versions.
119 
120  Returns
121  -------
122  packages : `dict`
123  Keys (type `str`) are package names; values (type `str`) are their versions.
124 
125  Notes
126  -----
127  We wade through sys.modules and attempt to determine the version for each
128  module. Note, therefore, that we can only report on modules that have
129  *already* been imported.
130 
131  We don't include any module for which we cannot determine a version.
132  """
133  # Attempt to import libraries that only report their version in python
134  for module in PYTHON:
135  try:
136  importlib.import_module(module)
137  except Exception:
138  pass # It's not available, so don't care
139 
140  packages = {"python": sys.version}
141  # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
142  moduleNames = list(sys.modules.keys())
143  for name in moduleNames:
144  module = sys.modules[name]
145  try:
146  ver = getVersionFromPythonModule(module)
147  except Exception:
148  continue # Can't get a version from it, don't care
149 
150  # Remove "foo.bar.version" in favor of "foo.bar"
151  # This prevents duplication when the __init__.py includes "from .version import *"
152  for ending in (".version", "._version"):
153  if name.endswith(ending):
154  name = name[:-len(ending)]
155  if name in packages:
156  assert ver == packages[name]
157  elif name in packages:
158  assert ver == packages[name]
159 
160  # Use LSST package names instead of python module names
161  # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
162  # versions if the environment reveals that we're not using the packages as-built.
163  if "lsst" in name:
164  name = name.replace("lsst.", "").replace(".", "_")
165 
166  packages[name] = ver
167 
168  return packages
169 
170 
171 _eups = None # Singleton Eups object
172 
173 
175  """Get products and their versions from the environment.
176 
177  Returns
178  -------
179  packages : `dict`
180  Keys (type `str`) are product names; values (type `str`) are their versions.
181 
182  Notes
183  -----
184  We use EUPS to determine the version of certain products (those that don't provide
185  a means to determine the version any other way) and to check if uninstalled packages
186  are being used. We only report the product/version for these packages.
187  """
188  try:
189  from eups import Eups
190  from eups.Product import Product
191  except ImportError:
192  from lsst.pex.logging import getDefaultLog
193  getDefaultLog().warn("Unable to import eups, so cannot determine package versions from environment")
194  return {}
195 
196  # Cache eups object since creating it can take a while
197  global _eups
198  if not _eups:
199  _eups = Eups()
200  products = _eups.findProducts(tags=["setup"])
201 
202  # Get versions for things we can't determine via runtime mechanisms
203  # XXX Should we just grab everything we can, rather than just a predetermined set?
204  packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
205 
206  # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
207  # code, so the version could be different than what's being reported by the runtime environment (because
208  # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
209  # probably doesn't check to see if the repo is clean).
210  for prod in products:
211  if not prod.version.startswith(Product.LocalVersionPrefix):
212  continue
213  ver = prod.version
214 
215  gitDir = os.path.join(prod.dir, ".git")
216  if os.path.exists(gitDir):
217  # get the git revision and an indication if the working copy is clean
218  revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
219  diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
220  "--patch"]
221  try:
222  rev = subprocess.check_output(revCmd).decode().strip()
223  diff = subprocess.check_output(diffCmd)
224  except Exception:
225  ver += "@GIT_ERROR"
226  else:
227  ver += "@" + rev
228  if diff:
229  ver += "+" + hashlib.md5(diff).hexdigest()
230  else:
231  ver += "@NO_GIT"
232 
233  packages[prod.name] = ver
234  return packages
235 
236 
237 class Packages(object):
238  """A table of packages and their versions.
239 
240  Parameters
241  ----------
242  packages : `dict`
243  A mapping {package: version} where both keys and values are type `str`.
244 
245  Notes
246  -----
247  This is essentially a wrapper around a dict with some conveniences.
248  """
249 
250  def __init__(self, packages):
251  assert isinstance(packages, Mapping)
252  self._packages = packages
253  self._names = set(packages.keys())
254 
255  @classmethod
256  def fromSystem(cls):
257  """Construct a `Packages` by examining the system.
258 
259  Determine packages by examining python's sys.modules, runtime libraries and EUPS.
260 
261  Returns
262  -------
263  packages : `Packages`
264  """
265  packages = {}
266  packages.update(getPythonPackages())
267  packages.update(getRuntimeVersions())
268  packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
269  return cls(packages)
270 
271  @classmethod
272  def read(cls, filename):
273  """Read packages from filename.
274 
275  Parameters
276  ----------
277  filename : `str`
278  Filename from which to read.
279 
280  Returns
281  -------
282  packages : `Packages`
283  """
284  with open(filename, "rb") as ff:
285  return pickle.load(ff)
286 
287  def write(self, filename):
288  """Write to file.
289 
290  Parameters
291  ----------
292  filename : `str`
293  Filename to which to write.
294  """
295  with open(filename, "wb") as ff:
296  pickle.dump(self, ff)
297 
298  def __len__(self):
299  return len(self._packages)
300 
301  def __str__(self):
302  ss = "%s({\n" % self.__class__.__name__
303  # Sort alphabetically by module name, for convenience in reading
304  ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
305  prod in sorted(self._names))
306  ss += ",\n})"
307  return ss
308 
309  def __repr__(self):
310  return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
311 
312  def __contains__(self, pkg):
313  return pkg in self._packages
314 
315  def __iter__(self):
316  return iter(self._packages)
317 
318  def update(self, other):
319  """Update packages with contents of another set of packages.
320 
321  Parameters
322  ----------
323  other : `Packages`
324  Other packages to merge with self.
325 
326  Notes
327  -----
328  No check is made to see if we're clobbering anything.
329  """
330  self._packages.update(other._packages)
331  self._names.update(other._names)
332 
333  def extra(self, other):
334  """Get packages in self but not in another `Packages` object.
335 
336  Parameters
337  ----------
338  other : `Packages`
339  Other packages to compare against.
340 
341  Returns
342  -------
343  extra : `dict`
344  Extra packages.
345  Keys (type `str`) are package names; values (type `str`) are their versions.
346  """
347  return {pkg: self._packages[pkg] for pkg in self._names - other._names}
348 
349  def missing(self, other):
350  """Get packages in another `Packages` object but missing from self.
351 
352  Parameters
353  ----------
354  other : `Packages`
355  Other packages to compare against.
356 
357  Returns
358  -------
359  missing : `dict`
360  Missing packages.
361  Keys (type `str`) are package names; values (type `str`) are their versions.
362  """
363  return {pkg: other._packages[pkg] for pkg in other._names - self._names}
364 
365  def difference(self, other):
366  """Get packages in symmetric difference of self and another `Packages` object.
367 
368  Parameters
369  ----------
370  other : `Packages`
371  Other packages to compare against.
372 
373  Returns
374  -------
375  difference : `dict`
376  Packages in symmetric difference.
377  Keys (type `str`) are package names; values (type `str`) are their versions.
378  """
379  return {pkg: (self._packages[pkg], other._packages[pkg]) for
380  pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}
def __init__(self, packages)
Definition: packages.py:250
def read(cls, filename)
Definition: packages.py:272
def write(self, filename)
Definition: packages.py:287
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54
def getEnvironmentPackages()
Definition: packages.py:174
def extra(self, other)
Definition: packages.py:333
def getVersionFromPythonModule(module)
Definition: packages.py:83
def __contains__(self, pkg)
Definition: packages.py:312
def update(self, other)
Definition: packages.py:318
def difference(self, other)
Definition: packages.py:365
def getPythonPackages()
Definition: packages.py:117
def missing(self, other)
Definition: packages.py:349