lsst.base  14.0
LSST Data Management Base Package
 All Classes Namespaces Files Functions Variables Friends Macros Pages
packages.py
Go to the documentation of this file.
1 """
2 Determine which packages are being used in the system and their versions
3 
4 There are a few different types of packages, and their versions are collected in different ways:
5 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from interrogating the dynamic library
6 2. Python modules (e.g., afw, numpy; galsim is also in this group even though we only use it through the
7  library, because no version information is currently provided through the library): we get their version
8  from the __version__ module variable. Note that this means that we're only aware of modules that have
9  already been imported.
10 3. Other packages provide no run-time accessible version information (e.g., astrometry_net): we get their
11  version from interrogating the environment. Currently, that means EUPS; if EUPS is replaced or dropped then
12  we'll need to consider an alternative means of getting this version information.
13 4. Local versions of packages (a non-installed EUPS package, selected with "setup -r /path/to/package"): we
14  identify these through the environment (EUPS again) and use as a version the path supplemented with the
15  git SHA and, if the git repo isn't clean, an MD5 of the diff.
16 
17 These package versions are collected and stored in a Packages object, which provides useful comparison and
18 persistence features.
19 
20 Example usage:
21 
22  from lsst.base import Packages
23  pkgs = Packages.fromSystem()
24  print "Current packages:", pkgs
25  old = Packages.read("/path/to/packages.pickle")
26  print "Old packages:", old
27  print "Missing packages compared to before:", pkgs.missing(old)
28  print "Extra packages compared to before:", pkgs.extra(old)
29  print "Different packages: ", pkgs.difference(old)
30  old.update(pkgs) # Include any new packages in the old
31  old.write("/path/to/packages.pickle")
32 """
33 from future import standard_library
34 standard_library.install_aliases()
35 from builtins import object
36 
37 import os
38 import sys
39 import hashlib
40 import importlib
41 import subprocess
42 import pickle as pickle
43 from collections import Mapping
44 
45 from .versions import getRuntimeVersions
46 
47 __all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
48 
49 
50 # Packages used at build-time (e.g., header-only)
51 BUILDTIME = set(["boost", "eigen", "tmv"])
52 
53 # Python modules to attempt to load so we can try to get the version
54 # We do this because the version only appears to be available from python, but we use the library
55 PYTHON = set(["galsim"])
56 
57 # Packages that don't seem to have a mechanism for reporting the runtime version
58 # We need to guess the version from the environment
59 ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
60 
61 
63  """Determine the version of a python module
64 
65  Will raise AttributeError if the __version__ attribute is not set.
66 
67  We supplement the version with information from the __dependency_versions__
68  (a specific variable set by LSST's sconsUtils at build time) only for packages
69  that are typically used only at build-time.
70  """
71  version = module.__version__
72  if hasattr(module, "__dependency_versions__"):
73  # Add build-time dependencies
74  deps = module.__dependency_versions__
75  buildtime = BUILDTIME & set(deps.keys())
76  if buildtime:
77  version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
78  for pkg in sorted(buildtime))
79  return version
80 
81 
83  """Return a dict of imported python packages and their versions
84 
85  We wade through sys.modules and attempt to determine the version for each
86  module. Note, therefore, that we can only report on modules that have
87  *already* been imported.
88 
89  We don't include any module for which we cannot determine a version.
90  """
91  # Attempt to import libraries that only report their version in python
92  for module in PYTHON:
93  try:
94  importlib.import_module(module)
95  except:
96  pass # It's not available, so don't care
97 
98  packages = {"python": sys.version}
99  # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
100  moduleNames = list(sys.modules.keys())
101  for name in moduleNames:
102  module = sys.modules[name]
103  try:
104  ver = getVersionFromPythonModule(module)
105  except:
106  continue # Can't get a version from it, don't care
107 
108  # Remove "foo.bar.version" in favor of "foo.bar"
109  # This prevents duplication when the __init__.py includes "from .version import *"
110  for ending in (".version", "._version"):
111  if name.endswith(ending):
112  name = name[:-len(ending)]
113  if name in packages:
114  assert ver == packages[name]
115  elif name in packages:
116  assert ver == packages[name]
117 
118  # Use LSST package names instead of python module names
119  # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
120  # versions if the environment reveals that we're not using the packages as-built.
121  if "lsst" in name:
122  name = name.replace("lsst.", "").replace(".", "_")
123 
124  packages[name] = ver
125 
126  return packages
127 
128 
129 _eups = None # Singleton Eups object
131  """Provide a dict of products and their versions from the environment
132 
133  We use EUPS to determine the version of certain products (those that don't provide
134  a means to determine the version any other way) and to check if uninstalled packages
135  are being used. We only report the product/version for these packages.
136  """
137  try:
138  from eups import Eups
139  from eups.Product import Product
140  except:
141  from lsst.pex.logging import getDefaultLog
142  getDefaultLog().warn("Unable to import eups, so cannot determine package versions from environment")
143  return {}
144 
145  # Cache eups object since creating it can take a while
146  global _eups
147  if not _eups:
148  _eups = Eups()
149  products = _eups.findProducts(tags=["setup"])
150 
151  # Get versions for things we can't determine via runtime mechanisms
152  # XXX Should we just grab everything we can, rather than just a predetermined set?
153  packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
154 
155  # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
156  # code, so the version could be different than what's being reported by the runtime environment (because
157  # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
158  # probably doesn't check to see if the repo is clean).
159  for prod in products:
160  if not prod.version.startswith(Product.LocalVersionPrefix):
161  continue
162  ver = prod.version
163 
164  gitDir = os.path.join(prod.dir, ".git")
165  if os.path.exists(gitDir):
166  # get the git revision and an indication if the working copy is clean
167  revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
168  diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
169  "--patch"]
170  try:
171  rev = subprocess.check_output(revCmd).decode().strip()
172  diff = subprocess.check_output(diffCmd)
173  except:
174  ver += "@GIT_ERROR"
175  else:
176  ver += "@" + rev
177  if diff:
178  ver += "+" + hashlib.md5(diff).hexdigest()
179  else:
180  ver += "@NO_GIT"
181 
182  packages[prod.name] = ver
183  return packages
184 
185 
186 class Packages(object):
187  """A table of packages and their versions
188 
189  Essentially a wrapper around a dict with some conveniences.
190  """
191 
192  def __init__(self, packages):
193  """Constructor
194 
195  'packages' should be a mapping {package: version}, such as a dict.
196  """
197  assert isinstance(packages, Mapping)
198  self._packages = packages
199  self._names = set(packages.keys())
200 
201  @classmethod
202  def fromSystem(cls):
203  """Construct from the system
204 
205  Attempts to determine packages by examining the system (python's sys.modules,
206  runtime libraries and EUPS).
207  """
208  packages = {}
209  packages.update(getPythonPackages())
210  packages.update(getRuntimeVersions())
211  packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
212  return cls(packages)
213 
214  @classmethod
215  def read(cls, filename):
216  """Read packages from filename"""
217  with open(filename, "rb") as ff:
218  return pickle.load(ff)
219 
220  def write(self, filename):
221  """Write packages to file"""
222  with open(filename, "wb") as ff:
223  pickle.dump(self, ff)
224 
225  def __len__(self):
226  return len(self._packages)
227 
228  def __str__(self):
229  ss = "%s({\n" % self.__class__.__name__
230  # Sort alphabetically by module name, for convenience in reading
231  ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
232  prod in sorted(self._names))
233  ss += ",\n})"
234  return ss
235 
236  def __repr__(self):
237  return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
238 
239  def __contains__(self, pkg):
240  return pkg in self._packages
241 
242  def __iter__(self):
243  return iter(self._packages)
244 
245  def update(self, other):
246  """Update packages using another set of packages
247 
248  No check is made to see if we're clobbering anything.
249  """
250  self._packages.update(other._packages)
251  self._names.update(other._names)
252 
253  def extra(self, other):
254  """Return packages in 'self' but not in 'other'
255 
256  These are extra packages in 'self' compared to 'other'.
257  """
258  return {pkg: self._packages[pkg] for pkg in self._names - other._names}
259 
260  def missing(self, other):
261  """Return packages in 'other' but not in 'self'
262 
263  These are missing packages in 'self' compared to 'other'.
264  """
265  return {pkg: other._packages[pkg] for pkg in other._names - self._names}
266 
267  def difference(self, other):
268  """Return packages different between 'self' and 'other'"""
269  return {pkg: (self._packages[pkg], other._packages[pkg]) for
270  pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}
def getEnvironmentPackages
Definition: packages.py:130
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54
def getVersionFromPythonModule
Definition: packages.py:62