lsst.base  13.0-12-g6586b4b+1
LSST Data Management Base Package
packages.py
Go to the documentation of this file.
1 """
2 Determine which packages are being used in the system and their versions
3 
4 There are a few different types of packages, and their versions are collected in different ways:
5 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from interrogating the dynamic library
6 2. Python modules (e.g., afw, numpy; galsim is also in this group even though we only use it through the
7  library, because no version information is currently provided through the library): we get their version
8  from the __version__ module variable. Note that this means that we're only aware of modules that have
9  already been imported.
10 3. Other packages provide no run-time accessible version information (e.g., astrometry_net): we get their
11  version from interrogating the environment. Currently, that means EUPS; if EUPS is replaced or dropped then
12  we'll need to consider an alternative means of getting this version information.
13 4. Local versions of packages (a non-installed EUPS package, selected with "setup -r /path/to/package"): we
14  identify these through the environment (EUPS again) and use as a version the path supplemented with the
15  git SHA and, if the git repo isn't clean, an MD5 of the diff.
16 
17 These package versions are collected and stored in a Packages object, which provides useful comparison and
18 persistence features.
19 
20 Example usage:
21 
22  from lsst.base import Packages
23  pkgs = Packages.fromSystem()
24  print "Current packages:", pkgs
25  old = Packages.read("/path/to/packages.pickle")
26  print "Old packages:", old
27  print "Missing packages compared to before:", pkgs.missing(old)
28  print "Extra packages compared to before:", pkgs.extra(old)
29  print "Different packages: ", pkgs.difference(old)
30  old.update(pkgs) # Include any new packages in the old
31  old.write("/path/to/packages.pickle")
32 """
33 from builtins import object
34 
35 import os
36 import sys
37 import hashlib
38 import importlib
39 import subprocess
40 import pickle as pickle
41 from collections import Mapping
42 
43 from .versions import getRuntimeVersions
44 
45 from future import standard_library
46 standard_library.install_aliases()
47 
48 __all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
49 
50 
51 # Packages used at build-time (e.g., header-only)
52 BUILDTIME = set(["boost", "eigen", "tmv"])
53 
54 # Python modules to attempt to load so we can try to get the version
55 # We do this because the version only appears to be available from python, but we use the library
56 PYTHON = set(["galsim"])
57 
58 # Packages that don't seem to have a mechanism for reporting the runtime version
59 # We need to guess the version from the environment
60 ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
61 
62 
64  """Determine the version of a python module
65 
66  Will raise AttributeError if the __version__ attribute is not set.
67 
68  We supplement the version with information from the __dependency_versions__
69  (a specific variable set by LSST's sconsUtils at build time) only for packages
70  that are typically used only at build-time.
71  """
72  version = module.__version__
73  if hasattr(module, "__dependency_versions__"):
74  # Add build-time dependencies
75  deps = module.__dependency_versions__
76  buildtime = BUILDTIME & set(deps.keys())
77  if buildtime:
78  version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
79  for pkg in sorted(buildtime))
80  return version
81 
82 
84  """Return a dict of imported python packages and their versions
85 
86  We wade through sys.modules and attempt to determine the version for each
87  module. Note, therefore, that we can only report on modules that have
88  *already* been imported.
89 
90  We don't include any module for which we cannot determine a version.
91  """
92  # Attempt to import libraries that only report their version in python
93  for module in PYTHON:
94  try:
95  importlib.import_module(module)
96  except:
97  pass # It's not available, so don't care
98 
99  packages = {"python": sys.version}
100  # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
101  moduleNames = list(sys.modules.keys())
102  for name in moduleNames:
103  module = sys.modules[name]
104  try:
105  ver = getVersionFromPythonModule(module)
106  except:
107  continue # Can't get a version from it, don't care
108 
109  # Remove "foo.bar.version" in favor of "foo.bar"
110  # This prevents duplication when the __init__.py includes "from .version import *"
111  for ending in (".version", "._version"):
112  if name.endswith(ending):
113  name = name[:-len(ending)]
114  if name in packages:
115  assert ver == packages[name]
116  elif name in packages:
117  assert ver == packages[name]
118 
119  # Use LSST package names instead of python module names
120  # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
121  # versions if the environment reveals that we're not using the packages as-built.
122  if "lsst" in name:
123  name = name.replace("lsst.", "").replace(".", "_")
124 
125  packages[name] = ver
126 
127  return packages
128 
129 
130 _eups = None # Singleton Eups object
131 
132 
134  """Provide a dict of products and their versions from the environment
135 
136  We use EUPS to determine the version of certain products (those that don't provide
137  a means to determine the version any other way) and to check if uninstalled packages
138  are being used. We only report the product/version for these packages.
139  """
140  try:
141  from eups import Eups
142  from eups.Product import Product
143  except:
144  from lsst.pex.logging import getDefaultLog
145  getDefaultLog().warn("Unable to import eups, so cannot determine package versions from environment")
146  return {}
147 
148  # Cache eups object since creating it can take a while
149  global _eups
150  if not _eups:
151  _eups = Eups()
152  products = _eups.findProducts(tags=["setup"])
153 
154  # Get versions for things we can't determine via runtime mechanisms
155  # XXX Should we just grab everything we can, rather than just a predetermined set?
156  packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
157 
158  # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
159  # code, so the version could be different than what's being reported by the runtime environment (because
160  # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
161  # probably doesn't check to see if the repo is clean).
162  for prod in products:
163  if not prod.version.startswith(Product.LocalVersionPrefix):
164  continue
165  ver = prod.version
166 
167  gitDir = os.path.join(prod.dir, ".git")
168  if os.path.exists(gitDir):
169  # get the git revision and an indication if the working copy is clean
170  revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
171  diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
172  "--patch"]
173  try:
174  rev = subprocess.check_output(revCmd).decode().strip()
175  diff = subprocess.check_output(diffCmd)
176  except:
177  ver += "@GIT_ERROR"
178  else:
179  ver += "@" + rev
180  if diff:
181  ver += "+" + hashlib.md5(diff).hexdigest()
182  else:
183  ver += "@NO_GIT"
184 
185  packages[prod.name] = ver
186  return packages
187 
188 
189 class Packages(object):
190  """A table of packages and their versions
191 
192  Essentially a wrapper around a dict with some conveniences.
193  """
194 
195  def __init__(self, packages):
196  """Constructor
197 
198  'packages' should be a mapping {package: version}, such as a dict.
199  """
200  assert isinstance(packages, Mapping)
201  self._packages = packages
202  self._names = set(packages.keys())
203 
204  @classmethod
205  def fromSystem(cls):
206  """Construct from the system
207 
208  Attempts to determine packages by examining the system (python's sys.modules,
209  runtime libraries and EUPS).
210  """
211  packages = {}
212  packages.update(getPythonPackages())
213  packages.update(getRuntimeVersions())
214  packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
215  return cls(packages)
216 
217  @classmethod
218  def read(cls, filename):
219  """Read packages from filename"""
220  with open(filename, "rb") as ff:
221  return pickle.load(ff)
222 
223  def write(self, filename):
224  """Write packages to file"""
225  with open(filename, "wb") as ff:
226  pickle.dump(self, ff)
227 
228  def __len__(self):
229  return len(self._packages)
230 
231  def __str__(self):
232  ss = "%s({\n" % self.__class__.__name__
233  # Sort alphabetically by module name, for convenience in reading
234  ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
235  prod in sorted(self._names))
236  ss += ",\n})"
237  return ss
238 
239  def __repr__(self):
240  return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
241 
242  def __contains__(self, pkg):
243  return pkg in self._packages
244 
245  def __iter__(self):
246  return iter(self._packages)
247 
248  def update(self, other):
249  """Update packages using another set of packages
250 
251  No check is made to see if we're clobbering anything.
252  """
253  self._packages.update(other._packages)
254  self._names.update(other._names)
255 
256  def extra(self, other):
257  """Return packages in 'self' but not in 'other'
258 
259  These are extra packages in 'self' compared to 'other'.
260  """
261  return {pkg: self._packages[pkg] for pkg in self._names - other._names}
262 
263  def missing(self, other):
264  """Return packages in 'other' but not in 'self'
265 
266  These are missing packages in 'self' compared to 'other'.
267  """
268  return {pkg: other._packages[pkg] for pkg in other._names - self._names}
269 
270  def difference(self, other):
271  """Return packages different between 'self' and 'other'"""
272  return {pkg: (self._packages[pkg], other._packages[pkg]) for
273  pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}
def __init__(self, packages)
Definition: packages.py:195
def read(cls, filename)
Definition: packages.py:218
def write(self, filename)
Definition: packages.py:223
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54
def getEnvironmentPackages()
Definition: packages.py:133
def extra(self, other)
Definition: packages.py:256
def getVersionFromPythonModule(module)
Definition: packages.py:63
def __contains__(self, pkg)
Definition: packages.py:242
def update(self, other)
Definition: packages.py:248
def difference(self, other)
Definition: packages.py:270
def getPythonPackages()
Definition: packages.py:83
def missing(self, other)
Definition: packages.py:263