2 Determine which packages are being used in the system and their versions
4 There are a few different types of packages, and their versions are collected in different ways:
5 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from interrogating the dynamic library
6 2. Python modules (e.g., afw, numpy; galsim is also in this group even though we only use it through the
7 library, because no version information is currently provided through the library): we get their version
8 from the __version__ module variable. Note that this means that we're only aware of modules that have
10 3. Other packages provide no run-time accessible version information (e.g., astrometry_net): we get their
11 version from interrogating the environment. Currently, that means EUPS; if EUPS is replaced or dropped then
12 we'll need to consider an alternative means of getting this version information.
13 4. Local versions of packages (a non-installed EUPS package, selected with "setup -r /path/to/package"): we
14 identify these through the environment (EUPS again) and use as a version the path supplemented with the
15 git SHA and, if the git repo isn't clean, an MD5 of the diff.
17 These package versions are collected and stored in a Packages object, which provides useful comparison and
22 from lsst.base import Packages
23 pkgs = Packages.fromSystem()
24 print "Current packages:", pkgs
25 old = Packages.read("/path/to/packages.pickle")
26 print "Old packages:", old
27 print "Missing packages compared to before:", pkgs.missing(old)
28 print "Extra packages compared to before:", pkgs.extra(old)
29 print "Different packages: ", pkgs.difference(old)
30 old.update(pkgs) # Include any new packages in the old
31 old.write("/path/to/packages.pickle")
33 from future
import standard_library
34 standard_library.install_aliases()
35 from builtins
import object
42 import pickle
as pickle
43 from collections
import Mapping
45 from .versions
import getRuntimeVersions
47 __all__ = [
"getVersionFromPythonModule",
"getPythonPackages",
"getEnvironmentPackages",
"Packages"]
51 BUILDTIME = set([
"boost",
"eigen",
"tmv"])
55 PYTHON = set([
"galsim"])
59 ENVIRONMENT = set([
"astrometry_net",
"astrometry_net_data",
"minuit2",
"xpa"])
63 """Determine the version of a python module
65 Will raise AttributeError if the __version__ attribute is not set.
67 We supplement the version with information from the __dependency_versions__
68 (a specific variable set by LSST's sconsUtils at build time) only for packages
69 that are typically used only at build-time.
71 version = module.__version__
72 if hasattr(module,
"__dependency_versions__"):
74 deps = module.__dependency_versions__
75 buildtime = BUILDTIME & set(deps.keys())
77 version +=
" with " +
" ".join(
"%s=%s" % (pkg, deps[pkg])
78 for pkg
in sorted(buildtime))
83 """Return a dict of imported python packages and their versions
85 We wade through sys.modules and attempt to determine the version for each
86 module. Note, therefore, that we can only report on modules that have
87 *already* been imported.
89 We don't include any module for which we cannot determine a version.
94 importlib.import_module(module)
98 packages = {
"python": sys.version}
100 moduleNames = list(sys.modules.keys())
101 for name
in moduleNames:
102 module = sys.modules[name]
110 for ending
in (
".version",
"._version"):
111 if name.endswith(ending):
112 name = name[:-len(ending)]
114 assert ver == packages[name]
115 elif name
in packages:
116 assert ver == packages[name]
122 name = name.replace(
"lsst.",
"").replace(
".",
"_")
131 """Provide a dict of products and their versions from the environment
133 We use EUPS to determine the version of certain products (those that don't provide
134 a means to determine the version any other way) and to check if uninstalled packages
135 are being used. We only report the product/version for these packages.
138 from eups
import Eups
139 from eups.Product
import Product
141 from lsst.pex.logging
import getDefaultLog
142 getDefaultLog().warn(
"Unable to import eups, so cannot determine package versions from environment")
149 products = _eups.findProducts(tags=[
"setup"])
153 packages = {prod.name: prod.version
for prod
in products
if prod
in ENVIRONMENT}
159 for prod
in products:
160 if not prod.version.startswith(Product.LocalVersionPrefix):
164 gitDir = os.path.join(prod.dir,
".git")
165 if os.path.exists(gitDir):
167 revCmd = [
"git",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"rev-parse",
"HEAD"]
168 diffCmd = [
"git",
"--no-pager",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"diff",
171 rev = subprocess.check_output(revCmd).decode().strip()
172 diff = subprocess.check_output(diffCmd)
178 ver +=
"+" + hashlib.md5(diff).hexdigest()
182 packages[prod.name] = ver
187 """A table of packages and their versions
189 Essentially a wrapper around a dict with some conveniences.
195 'packages' should be a mapping {package: version}, such as a dict.
197 assert isinstance(packages, Mapping)
199 self.
_names = set(packages.keys())
203 """Construct from the system
205 Attempts to determine packages by examining the system (python's sys.modules,
206 runtime libraries and EUPS).
216 """Read packages from filename"""
217 with open(filename,
"rb")
as ff:
218 return pickle.load(ff)
221 """Write packages to file"""
222 with open(filename,
"wb")
as ff:
223 pickle.dump(self, ff)
229 ss =
"%s({\n" % self.__class__.__name__
231 ss +=
",\n".join(
"%s: %s" % (repr(prod), repr(self.
_packages[prod]))
for
232 prod
in sorted(self.
_names))
237 return "%s(%s)" % (self.__class__.__name__, repr(self.
_packages))
246 """Update packages using another set of packages
248 No check is made to see if we're clobbering anything.
250 self._packages.update(other._packages)
251 self._names.update(other._names)
254 """Return packages in 'self' but not in 'other'
256 These are extra packages in 'self' compared to 'other'.
258 return {pkg: self.
_packages[pkg]
for pkg
in self.
_names - other._names}
261 """Return packages in 'other' but not in 'self'
263 These are missing packages in 'self' compared to 'other'.
265 return {pkg: other._packages[pkg]
for pkg
in other._names - self.
_names}
268 """Return packages different between 'self' and 'other'"""
269 return {pkg: (self.
_packages[pkg], other._packages[pkg])
for
270 pkg
in self.
_names & other._names
if self.
_packages[pkg] != other._packages[pkg]}
def getEnvironmentPackages
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
def getVersionFromPythonModule