lsst.base  16.0-2-g55907dc+1
LSST Data Management Base Package
packages.py
Go to the documentation of this file.
1 # This file is part of base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """
22 Determine which packages are being used in the system and their versions
23 """
24 import os
25 import sys
26 import hashlib
27 import importlib
28 import subprocess
29 import pickle as pickle
30 from collections import Mapping
31 
32 from .versions import getRuntimeVersions
33 
34 __all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
35 
36 
37 # Packages used at build-time (e.g., header-only)
38 BUILDTIME = set(["boost", "eigen", "tmv"])
39 
40 # Python modules to attempt to load so we can try to get the version
41 # We do this because the version only appears to be available from python, but we use the library
42 PYTHON = set(["galsim"])
43 
44 # Packages that don't seem to have a mechanism for reporting the runtime version
45 # We need to guess the version from the environment
46 ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
47 
48 
50  """Determine the version of a python module.
51 
52  Parameters
53  ----------
54  module : `module`
55  Module for which to get version.
56 
57  Returns
58  -------
59  version : `str`
60 
61  Raises
62  ------
63  AttributeError
64  Raised if __version__ attribute is not set.
65 
66  Notes
67  -----
68  We supplement the version with information from the
69  ``__dependency_versions__`` (a specific variable set by LSST's
70  `~lsst.sconsUtils` at build time) only for packages that are typically
71  used only at build-time.
72  """
73  version = module.__version__
74  if hasattr(module, "__dependency_versions__"):
75  # Add build-time dependencies
76  deps = module.__dependency_versions__
77  buildtime = BUILDTIME & set(deps.keys())
78  if buildtime:
79  version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
80  for pkg in sorted(buildtime))
81  return version
82 
83 
85  """Get imported python packages and their versions.
86 
87  Returns
88  -------
89  packages : `dict`
90  Keys (type `str`) are package names; values (type `str`) are their
91  versions.
92 
93  Notes
94  -----
95  We wade through `sys.modules` and attempt to determine the version for each
96  module. Note, therefore, that we can only report on modules that have
97  *already* been imported.
98 
99  We don't include any module for which we cannot determine a version.
100  """
101  # Attempt to import libraries that only report their version in python
102  for module in PYTHON:
103  try:
104  importlib.import_module(module)
105  except Exception:
106  pass # It's not available, so don't care
107 
108  packages = {"python": sys.version}
109  # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
110  moduleNames = list(sys.modules.keys())
111  for name in moduleNames:
112  module = sys.modules[name]
113  try:
114  ver = getVersionFromPythonModule(module)
115  except Exception:
116  continue # Can't get a version from it, don't care
117 
118  # Remove "foo.bar.version" in favor of "foo.bar"
119  # This prevents duplication when the __init__.py includes "from .version import *"
120  for ending in (".version", "._version"):
121  if name.endswith(ending):
122  name = name[:-len(ending)]
123  if name in packages:
124  assert ver == packages[name]
125  elif name in packages:
126  assert ver == packages[name]
127 
128  # Use LSST package names instead of python module names
129  # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
130  # versions if the environment reveals that we're not using the packages as-built.
131  if "lsst" in name:
132  name = name.replace("lsst.", "").replace(".", "_")
133 
134  packages[name] = ver
135 
136  return packages
137 
138 
139 _eups = None # Singleton Eups object
140 
141 
143  """Get products and their versions from the environment.
144 
145  Returns
146  -------
147  packages : `dict`
148  Keys (type `str`) are product names; values (type `str`) are their
149  versions.
150 
151  Notes
152  -----
153  We use EUPS to determine the version of certain products (those that don't
154  provide a means to determine the version any other way) and to check if
155  uninstalled packages are being used. We only report the product/version
156  for these packages.
157  """
158  try:
159  from eups import Eups
160  from eups.Product import Product
161  except ImportError:
162  from lsst.pex.logging import getDefaultLog
163  getDefaultLog().warn("Unable to import eups, so cannot determine package versions from environment")
164  return {}
165 
166  # Cache eups object since creating it can take a while
167  global _eups
168  if not _eups:
169  _eups = Eups()
170  products = _eups.findProducts(tags=["setup"])
171 
172  # Get versions for things we can't determine via runtime mechanisms
173  # XXX Should we just grab everything we can, rather than just a predetermined set?
174  packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
175 
176  # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
177  # code, so the version could be different than what's being reported by the runtime environment (because
178  # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
179  # probably doesn't check to see if the repo is clean).
180  for prod in products:
181  if not prod.version.startswith(Product.LocalVersionPrefix):
182  continue
183  ver = prod.version
184 
185  gitDir = os.path.join(prod.dir, ".git")
186  if os.path.exists(gitDir):
187  # get the git revision and an indication if the working copy is clean
188  revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
189  diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
190  "--patch"]
191  try:
192  rev = subprocess.check_output(revCmd).decode().strip()
193  diff = subprocess.check_output(diffCmd)
194  except Exception:
195  ver += "@GIT_ERROR"
196  else:
197  ver += "@" + rev
198  if diff:
199  ver += "+" + hashlib.md5(diff).hexdigest()
200  else:
201  ver += "@NO_GIT"
202 
203  packages[prod.name] = ver
204  return packages
205 
206 
207 class Packages:
208  """A table of packages and their versions.
209 
210  There are a few different types of packages, and their versions are collected
211  in different ways:
212 
213  1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
214  interrogating the dynamic library
215  2. Python modules (e.g., afw, numpy; galsim is also in this group even though
216  we only use it through the library, because no version information is
217  currently provided through the library): we get their version from the
218  ``__version__`` module variable. Note that this means that we're only aware
219  of modules that have already been imported.
220  3. Other packages provide no run-time accessible version information (e.g.,
221  astrometry_net): we get their version from interrogating the environment.
222  Currently, that means EUPS; if EUPS is replaced or dropped then we'll need
223  to consider an alternative means of getting this version information.
224  4. Local versions of packages (a non-installed EUPS package, selected with
225  ``setup -r /path/to/package``): we identify these through the environment
226  (EUPS again) and use as a version the path supplemented with the ``git``
227  SHA and, if the git repo isn't clean, an MD5 of the diff.
228 
229  These package versions are collected and stored in a Packages object, which
230  provides useful comparison and persistence features.
231 
232  Example usage:
233 
234  .. code-block:: python
235 
236  from lsst.base import Packages
237  pkgs = Packages.fromSystem()
238  print("Current packages:", pkgs)
239  old = Packages.read("/path/to/packages.pickle")
240  print("Old packages:", old)
241  print("Missing packages compared to before:", pkgs.missing(old))
242  print("Extra packages compared to before:", pkgs.extra(old))
243  print("Different packages: ", pkgs.difference(old))
244  old.update(pkgs) # Include any new packages in the old
245  old.write("/path/to/packages.pickle")
246 
247  Parameters
248  ----------
249  packages : `dict`
250  A mapping {package: version} where both keys and values are type `str`.
251 
252  Notes
253  -----
254  This is essentially a wrapper around a dict with some conveniences.
255  """
256 
257  def __init__(self, packages):
258  assert isinstance(packages, Mapping)
259  self._packages = packages
260  self._names = set(packages.keys())
261 
262  @classmethod
263  def fromSystem(cls):
264  """Construct a `Packages` by examining the system.
265 
266  Determine packages by examining python's `sys.modules`, runtime
267  libraries and EUPS.
268 
269  Returns
270  -------
271  packages : `Packages`
272  """
273  packages = {}
274  packages.update(getPythonPackages())
275  packages.update(getRuntimeVersions())
276  packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
277  return cls(packages)
278 
279  @classmethod
280  def read(cls, filename):
281  """Read packages from filename.
282 
283  Parameters
284  ----------
285  filename : `str`
286  Filename from which to read.
287 
288  Returns
289  -------
290  packages : `Packages`
291  """
292  with open(filename, "rb") as ff:
293  return pickle.load(ff)
294 
295  def write(self, filename):
296  """Write to file.
297 
298  Parameters
299  ----------
300  filename : `str`
301  Filename to which to write.
302  """
303  with open(filename, "wb") as ff:
304  pickle.dump(self, ff)
305 
306  def __len__(self):
307  return len(self._packages)
308 
309  def __str__(self):
310  ss = "%s({\n" % self.__class__.__name__
311  # Sort alphabetically by module name, for convenience in reading
312  ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
313  prod in sorted(self._names))
314  ss += ",\n})"
315  return ss
316 
317  def __repr__(self):
318  return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
319 
320  def __contains__(self, pkg):
321  return pkg in self._packages
322 
323  def __iter__(self):
324  return iter(self._packages)
325 
326  def update(self, other):
327  """Update packages with contents of another set of packages.
328 
329  Parameters
330  ----------
331  other : `Packages`
332  Other packages to merge with self.
333 
334  Notes
335  -----
336  No check is made to see if we're clobbering anything.
337  """
338  self._packages.update(other._packages)
339  self._names.update(other._names)
340 
341  def extra(self, other):
342  """Get packages in self but not in another `Packages` object.
343 
344  Parameters
345  ----------
346  other : `Packages`
347  Other packages to compare against.
348 
349  Returns
350  -------
351  extra : `dict`
352  Extra packages. Keys (type `str`) are package names; values
353  (type `str`) are their versions.
354  """
355  return {pkg: self._packages[pkg] for pkg in self._names - other._names}
356 
357  def missing(self, other):
358  """Get packages in another `Packages` object but missing from self.
359 
360  Parameters
361  ----------
362  other : `Packages`
363  Other packages to compare against.
364 
365  Returns
366  -------
367  missing : `dict`
368  Missing packages. Keys (type `str`) are package names; values
369  (type `str`) are their versions.
370  """
371  return {pkg: other._packages[pkg] for pkg in other._names - self._names}
372 
373  def difference(self, other):
374  """Get packages in symmetric difference of self and another `Packages`
375  object.
376 
377  Parameters
378  ----------
379  other : `Packages`
380  Other packages to compare against.
381 
382  Returns
383  -------
384  difference : `dict`
385  Packages in symmetric difference. Keys (type `str`) are package
386  names; values (type `str`) are their versions.
387  """
388  return {pkg: (self._packages[pkg], other._packages[pkg]) for
389  pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}
def __init__(self, packages)
Definition: packages.py:257
def read(cls, filename)
Definition: packages.py:280
def write(self, filename)
Definition: packages.py:295
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54
def getEnvironmentPackages()
Definition: packages.py:142
def extra(self, other)
Definition: packages.py:341
def getVersionFromPythonModule(module)
Definition: packages.py:49
def __contains__(self, pkg)
Definition: packages.py:320
def update(self, other)
Definition: packages.py:326
def difference(self, other)
Definition: packages.py:373
def getPythonPackages()
Definition: packages.py:84
def missing(self, other)
Definition: packages.py:357