lsst.base  19.0.0-6-g2334185
LSST Data Management Base Package
packages.py
Go to the documentation of this file.
1 # This file is part of base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """
22 Determine which packages are being used in the system and their versions
23 """
24 import os
25 import sys
26 import hashlib
27 import importlib
28 import subprocess
29 import logging
30 import pickle as pickle
31 import yaml
32 from collections.abc import Mapping
33 
34 from .versions import getRuntimeVersions
35 
36 log = logging.getLogger(__name__)
37 
38 __all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
39 
40 
41 # Packages used at build-time (e.g., header-only)
42 BUILDTIME = set(["boost", "eigen", "tmv"])
43 
44 # Python modules to attempt to load so we can try to get the version
45 # We do this because the version only appears to be available from python, but we use the library
46 PYTHON = set(["galsim"])
47 
48 # Packages that don't seem to have a mechanism for reporting the runtime version
49 # We need to guess the version from the environment
50 ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
51 
52 
54  """Determine the version of a python module.
55 
56  Parameters
57  ----------
58  module : `module`
59  Module for which to get version.
60 
61  Returns
62  -------
63  version : `str`
64 
65  Raises
66  ------
67  AttributeError
68  Raised if __version__ attribute is not set.
69 
70  Notes
71  -----
72  We supplement the version with information from the
73  ``__dependency_versions__`` (a specific variable set by LSST's
74  `~lsst.sconsUtils` at build time) only for packages that are typically
75  used only at build-time.
76  """
77  version = module.__version__
78  if hasattr(module, "__dependency_versions__"):
79  # Add build-time dependencies
80  deps = module.__dependency_versions__
81  buildtime = BUILDTIME & set(deps.keys())
82  if buildtime:
83  version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
84  for pkg in sorted(buildtime))
85  return version
86 
87 
89  """Get imported python packages and their versions.
90 
91  Returns
92  -------
93  packages : `dict`
94  Keys (type `str`) are package names; values (type `str`) are their
95  versions.
96 
97  Notes
98  -----
99  We wade through `sys.modules` and attempt to determine the version for each
100  module. Note, therefore, that we can only report on modules that have
101  *already* been imported.
102 
103  We don't include any module for which we cannot determine a version.
104  """
105  # Attempt to import libraries that only report their version in python
106  for module in PYTHON:
107  try:
108  importlib.import_module(module)
109  except Exception:
110  pass # It's not available, so don't care
111 
112  packages = {"python": sys.version}
113  # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
114  moduleNames = list(sys.modules.keys())
115  for name in moduleNames:
116  module = sys.modules[name]
117  try:
118  ver = getVersionFromPythonModule(module)
119  except Exception:
120  continue # Can't get a version from it, don't care
121 
122  # Remove "foo.bar.version" in favor of "foo.bar"
123  # This prevents duplication when the __init__.py includes "from .version import *"
124  for ending in (".version", "._version"):
125  if name.endswith(ending):
126  name = name[:-len(ending)]
127  if name in packages:
128  assert ver == packages[name]
129  elif name in packages:
130  assert ver == packages[name]
131 
132  # Use LSST package names instead of python module names
133  # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
134  # versions if the environment reveals that we're not using the packages as-built.
135  if "lsst" in name:
136  name = name.replace("lsst.", "").replace(".", "_")
137 
138  packages[name] = ver
139 
140  return packages
141 
142 
143 _eups = None # Singleton Eups object
144 
145 
147  """Get products and their versions from the environment.
148 
149  Returns
150  -------
151  packages : `dict`
152  Keys (type `str`) are product names; values (type `str`) are their
153  versions.
154 
155  Notes
156  -----
157  We use EUPS to determine the version of certain products (those that don't
158  provide a means to determine the version any other way) and to check if
159  uninstalled packages are being used. We only report the product/version
160  for these packages.
161  """
162  try:
163  from eups import Eups
164  from eups.Product import Product
165  except ImportError:
166  log.warning("Unable to import eups, so cannot determine package versions from environment")
167  return {}
168 
169  # Cache eups object since creating it can take a while
170  global _eups
171  if not _eups:
172  _eups = Eups()
173  products = _eups.findProducts(tags=["setup"])
174 
175  # Get versions for things we can't determine via runtime mechanisms
176  # XXX Should we just grab everything we can, rather than just a predetermined set?
177  packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
178 
179  # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
180  # code, so the version could be different than what's being reported by the runtime environment (because
181  # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
182  # probably doesn't check to see if the repo is clean).
183  for prod in products:
184  if not prod.version.startswith(Product.LocalVersionPrefix):
185  continue
186  ver = prod.version
187 
188  gitDir = os.path.join(prod.dir, ".git")
189  if os.path.exists(gitDir):
190  # get the git revision and an indication if the working copy is clean
191  revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
192  diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
193  "--patch"]
194  try:
195  rev = subprocess.check_output(revCmd).decode().strip()
196  diff = subprocess.check_output(diffCmd)
197  except Exception:
198  ver += "@GIT_ERROR"
199  else:
200  ver += "@" + rev
201  if diff:
202  ver += "+" + hashlib.md5(diff).hexdigest()
203  else:
204  ver += "@NO_GIT"
205 
206  packages[prod.name] = ver
207  return packages
208 
209 
210 class Packages:
211  """A table of packages and their versions.
212 
213  There are a few different types of packages, and their versions are collected
214  in different ways:
215 
216  1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
217  interrogating the dynamic library
218  2. Python modules (e.g., afw, numpy; galsim is also in this group even though
219  we only use it through the library, because no version information is
220  currently provided through the library): we get their version from the
221  ``__version__`` module variable. Note that this means that we're only aware
222  of modules that have already been imported.
223  3. Other packages provide no run-time accessible version information (e.g.,
224  astrometry_net): we get their version from interrogating the environment.
225  Currently, that means EUPS; if EUPS is replaced or dropped then we'll need
226  to consider an alternative means of getting this version information.
227  4. Local versions of packages (a non-installed EUPS package, selected with
228  ``setup -r /path/to/package``): we identify these through the environment
229  (EUPS again) and use as a version the path supplemented with the ``git``
230  SHA and, if the git repo isn't clean, an MD5 of the diff.
231 
232  These package versions are collected and stored in a Packages object, which
233  provides useful comparison and persistence features.
234 
235  Example usage:
236 
237  .. code-block:: python
238 
239  from lsst.base import Packages
240  pkgs = Packages.fromSystem()
241  print("Current packages:", pkgs)
242  old = Packages.read("/path/to/packages.pickle")
243  print("Old packages:", old)
244  print("Missing packages compared to before:", pkgs.missing(old))
245  print("Extra packages compared to before:", pkgs.extra(old))
246  print("Different packages: ", pkgs.difference(old))
247  old.update(pkgs) # Include any new packages in the old
248  old.write("/path/to/packages.pickle")
249 
250  Parameters
251  ----------
252  packages : `dict`
253  A mapping {package: version} where both keys and values are type `str`.
254 
255  Notes
256  -----
257  This is essentially a wrapper around a dict with some conveniences.
258  """
259 
260  def __init__(self, packages):
261  assert isinstance(packages, Mapping)
262  self._packages = packages
263  self._names = set(packages.keys())
264 
265  @classmethod
266  def fromSystem(cls):
267  """Construct a `Packages` by examining the system.
268 
269  Determine packages by examining python's `sys.modules`, runtime
270  libraries and EUPS.
271 
272  Returns
273  -------
274  packages : `Packages`
275  """
276  packages = {}
277  packages.update(getPythonPackages())
278  packages.update(getRuntimeVersions())
279  packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
280  return cls(packages)
281 
282  @classmethod
283  def read(cls, filename):
284  """Read packages from filename.
285 
286  Parameters
287  ----------
288  filename : `str`
289  Filename from which to read. The format is determined from the
290  file extension. Currently support ``.pickle``, ``.pkl``
291  and ``.yaml``.
292 
293  Returns
294  -------
295  packages : `Packages`
296  """
297  _, ext = os.path.splitext(filename)
298  if ext in (".pickle", ".pkl"):
299  with open(filename, "rb") as ff:
300  return pickle.load(ff)
301  elif ext == ".yaml":
302  with open(filename, "r") as ff:
303  return yaml.load(ff, Loader=yaml.SafeLoader)
304  else:
305  raise ValueError(f"Unable to determine how to read file {filename} from extension {ext}")
306 
307  def write(self, filename):
308  """Write to file.
309 
310  Parameters
311  ----------
312  filename : `str`
313  Filename to which to write. The format of the data file
314  is determined from the file extension. Currently supports
315  ``.pickle`` and ``.yaml``
316  """
317  _, ext = os.path.splitext(filename)
318  if ext in (".pickle", ".pkl"):
319  with open(filename, "wb") as ff:
320  pickle.dump(self, ff)
321  elif ext == ".yaml":
322  with open(filename, "w") as ff:
323  yaml.dump(self, ff)
324  else:
325  raise ValueError(f"Unexpected file format requested: {ext}")
326 
327  def __len__(self):
328  return len(self._packages)
329 
330  def __str__(self):
331  ss = "%s({\n" % self.__class__.__name__
332  # Sort alphabetically by module name, for convenience in reading
333  ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
334  prod in sorted(self._names))
335  ss += ",\n})"
336  return ss
337 
338  def __repr__(self):
339  return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
340 
341  def __contains__(self, pkg):
342  return pkg in self._packages
343 
344  def __iter__(self):
345  return iter(self._packages)
346 
347  def __eq__(self, other):
348  if not isinstance(other, type(self)):
349  return False
350 
351  return self._packages == other._packages
352 
353  def update(self, other):
354  """Update packages with contents of another set of packages.
355 
356  Parameters
357  ----------
358  other : `Packages`
359  Other packages to merge with self.
360 
361  Notes
362  -----
363  No check is made to see if we're clobbering anything.
364  """
365  self._packages.update(other._packages)
366  self._names.update(other._names)
367 
368  def extra(self, other):
369  """Get packages in self but not in another `Packages` object.
370 
371  Parameters
372  ----------
373  other : `Packages`
374  Other packages to compare against.
375 
376  Returns
377  -------
378  extra : `dict`
379  Extra packages. Keys (type `str`) are package names; values
380  (type `str`) are their versions.
381  """
382  return {pkg: self._packages[pkg] for pkg in self._names - other._names}
383 
384  def missing(self, other):
385  """Get packages in another `Packages` object but missing from self.
386 
387  Parameters
388  ----------
389  other : `Packages`
390  Other packages to compare against.
391 
392  Returns
393  -------
394  missing : `dict`
395  Missing packages. Keys (type `str`) are package names; values
396  (type `str`) are their versions.
397  """
398  return {pkg: other._packages[pkg] for pkg in other._names - self._names}
399 
400  def difference(self, other):
401  """Get packages in symmetric difference of self and another `Packages`
402  object.
403 
404  Parameters
405  ----------
406  other : `Packages`
407  Other packages to compare against.
408 
409  Returns
410  -------
411  difference : `dict`
412  Packages in symmetric difference. Keys (type `str`) are package
413  names; values (type `str`) are their versions.
414  """
415  return {pkg: (self._packages[pkg], other._packages[pkg]) for
416  pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}
417 
418 
419 # Register YAML representers
420 
421 def pkg_representer(dumper, data):
422  """Represent Packages as a simple dict"""
423  return dumper.represent_mapping("lsst.base.Packages", data._packages,
424  flow_style=None)
425 
426 
427 yaml.add_representer(Packages, pkg_representer)
428 
429 
430 def pkg_constructor(loader, node):
431  yield Packages(loader.construct_mapping(node, deep=True))
432 
433 
434 for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
435  yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)
base.packages.Packages.fromSystem
def fromSystem(cls)
Definition: packages.py:266
base.packages.Packages.__str__
def __str__(self)
Definition: packages.py:330
base.packages.Packages._names
_names
Definition: packages.py:263
base.packages.getPythonPackages
def getPythonPackages()
Definition: packages.py:88
base.packages.Packages.update
def update(self, other)
Definition: packages.py:353
base.packages.Packages.missing
def missing(self, other)
Definition: packages.py:384
base.packages.Packages.__contains__
def __contains__(self, pkg)
Definition: packages.py:341
lsst::base::getRuntimeVersions
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54
base.packages.pkg_constructor
pkg_constructor
Definition: packages.py:435
base.packages.Packages.__repr__
def __repr__(self)
Definition: packages.py:338
base.packages.Packages.__len__
def __len__(self)
Definition: packages.py:327
base.packages.Packages
Definition: packages.py:210
base.packages.Packages.extra
def extra(self, other)
Definition: packages.py:368
base.packages.pkg_representer
def pkg_representer(dumper, data)
Definition: packages.py:421
base.packages.Packages.__eq__
def __eq__(self, other)
Definition: packages.py:347
base.packages.getVersionFromPythonModule
def getVersionFromPythonModule(module)
Definition: packages.py:53
base.packages.Packages.__init__
def __init__(self, packages)
Definition: packages.py:260
base.packages.getEnvironmentPackages
def getEnvironmentPackages()
Definition: packages.py:146
base.packages.Packages.read
def read(cls, filename)
Definition: packages.py:283
base.packages.Packages._packages
_packages
Definition: packages.py:262
base.packages.Packages.difference
def difference(self, other)
Definition: packages.py:400
base.packages.Packages.__iter__
def __iter__(self)
Definition: packages.py:344
base.packages.Packages.write
def write(self, filename)
Definition: packages.py:307