Coverage for python/lsst/base/packages.py : 21%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""
22Determine which packages are being used in the system and their versions
23"""
24import os
25import sys
26import hashlib
27import importlib
28import subprocess
29import logging
30import pickle as pickle
31from collections.abc import Mapping
33from .versions import getRuntimeVersions
35log = logging.getLogger(__name__)
37__all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
40# Packages used at build-time (e.g., header-only)
41BUILDTIME = set(["boost", "eigen", "tmv"])
43# Python modules to attempt to load so we can try to get the version
44# We do this because the version only appears to be available from python, but we use the library
45PYTHON = set(["galsim"])
47# Packages that don't seem to have a mechanism for reporting the runtime version
48# We need to guess the version from the environment
49ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
52def getVersionFromPythonModule(module):
53 """Determine the version of a python module.
55 Parameters
56 ----------
57 module : `module`
58 Module for which to get version.
60 Returns
61 -------
62 version : `str`
64 Raises
65 ------
66 AttributeError
67 Raised if __version__ attribute is not set.
69 Notes
70 -----
71 We supplement the version with information from the
72 ``__dependency_versions__`` (a specific variable set by LSST's
73 `~lsst.sconsUtils` at build time) only for packages that are typically
74 used only at build-time.
75 """
76 version = module.__version__
77 if hasattr(module, "__dependency_versions__"):
78 # Add build-time dependencies
79 deps = module.__dependency_versions__
80 buildtime = BUILDTIME & set(deps.keys())
81 if buildtime:
82 version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
83 for pkg in sorted(buildtime))
84 return version
87def getPythonPackages():
88 """Get imported python packages and their versions.
90 Returns
91 -------
92 packages : `dict`
93 Keys (type `str`) are package names; values (type `str`) are their
94 versions.
96 Notes
97 -----
98 We wade through `sys.modules` and attempt to determine the version for each
99 module. Note, therefore, that we can only report on modules that have
100 *already* been imported.
102 We don't include any module for which we cannot determine a version.
103 """
104 # Attempt to import libraries that only report their version in python
105 for module in PYTHON:
106 try:
107 importlib.import_module(module)
108 except Exception:
109 pass # It's not available, so don't care
111 packages = {"python": sys.version}
112 # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
113 moduleNames = list(sys.modules.keys())
114 for name in moduleNames:
115 module = sys.modules[name]
116 try:
117 ver = getVersionFromPythonModule(module)
118 except Exception:
119 continue # Can't get a version from it, don't care
121 # Remove "foo.bar.version" in favor of "foo.bar"
122 # This prevents duplication when the __init__.py includes "from .version import *"
123 for ending in (".version", "._version"):
124 if name.endswith(ending):
125 name = name[:-len(ending)]
126 if name in packages:
127 assert ver == packages[name]
128 elif name in packages:
129 assert ver == packages[name]
131 # Use LSST package names instead of python module names
132 # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
133 # versions if the environment reveals that we're not using the packages as-built.
134 if "lsst" in name:
135 name = name.replace("lsst.", "").replace(".", "_")
137 packages[name] = ver
139 return packages
142_eups = None # Singleton Eups object
145def getEnvironmentPackages():
146 """Get products and their versions from the environment.
148 Returns
149 -------
150 packages : `dict`
151 Keys (type `str`) are product names; values (type `str`) are their
152 versions.
154 Notes
155 -----
156 We use EUPS to determine the version of certain products (those that don't
157 provide a means to determine the version any other way) and to check if
158 uninstalled packages are being used. We only report the product/version
159 for these packages.
160 """
161 try:
162 from eups import Eups
163 from eups.Product import Product
164 except ImportError:
165 log.warning("Unable to import eups, so cannot determine package versions from environment")
166 return {}
168 # Cache eups object since creating it can take a while
169 global _eups
170 if not _eups:
171 _eups = Eups()
172 products = _eups.findProducts(tags=["setup"])
174 # Get versions for things we can't determine via runtime mechanisms
175 # XXX Should we just grab everything we can, rather than just a predetermined set?
176 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
178 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
179 # code, so the version could be different than what's being reported by the runtime environment (because
180 # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
181 # probably doesn't check to see if the repo is clean).
182 for prod in products:
183 if not prod.version.startswith(Product.LocalVersionPrefix):
184 continue
185 ver = prod.version
187 gitDir = os.path.join(prod.dir, ".git")
188 if os.path.exists(gitDir):
189 # get the git revision and an indication if the working copy is clean
190 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
191 diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
192 "--patch"]
193 try:
194 rev = subprocess.check_output(revCmd).decode().strip()
195 diff = subprocess.check_output(diffCmd)
196 except Exception:
197 ver += "@GIT_ERROR"
198 else:
199 ver += "@" + rev
200 if diff:
201 ver += "+" + hashlib.md5(diff).hexdigest()
202 else:
203 ver += "@NO_GIT"
205 packages[prod.name] = ver
206 return packages
209class Packages:
210 """A table of packages and their versions.
212 There are a few different types of packages, and their versions are collected
213 in different ways:
215 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
216 interrogating the dynamic library
217 2. Python modules (e.g., afw, numpy; galsim is also in this group even though
218 we only use it through the library, because no version information is
219 currently provided through the library): we get their version from the
220 ``__version__`` module variable. Note that this means that we're only aware
221 of modules that have already been imported.
222 3. Other packages provide no run-time accessible version information (e.g.,
223 astrometry_net): we get their version from interrogating the environment.
224 Currently, that means EUPS; if EUPS is replaced or dropped then we'll need
225 to consider an alternative means of getting this version information.
226 4. Local versions of packages (a non-installed EUPS package, selected with
227 ``setup -r /path/to/package``): we identify these through the environment
228 (EUPS again) and use as a version the path supplemented with the ``git``
229 SHA and, if the git repo isn't clean, an MD5 of the diff.
231 These package versions are collected and stored in a Packages object, which
232 provides useful comparison and persistence features.
234 Example usage:
236 .. code-block:: python
238 from lsst.base import Packages
239 pkgs = Packages.fromSystem()
240 print("Current packages:", pkgs)
241 old = Packages.read("/path/to/packages.pickle")
242 print("Old packages:", old)
243 print("Missing packages compared to before:", pkgs.missing(old))
244 print("Extra packages compared to before:", pkgs.extra(old))
245 print("Different packages: ", pkgs.difference(old))
246 old.update(pkgs) # Include any new packages in the old
247 old.write("/path/to/packages.pickle")
249 Parameters
250 ----------
251 packages : `dict`
252 A mapping {package: version} where both keys and values are type `str`.
254 Notes
255 -----
256 This is essentially a wrapper around a dict with some conveniences.
257 """
259 def __init__(self, packages):
260 assert isinstance(packages, Mapping)
261 self._packages = packages
262 self._names = set(packages.keys())
264 @classmethod
265 def fromSystem(cls):
266 """Construct a `Packages` by examining the system.
268 Determine packages by examining python's `sys.modules`, runtime
269 libraries and EUPS.
271 Returns
272 -------
273 packages : `Packages`
274 """
275 packages = {}
276 packages.update(getPythonPackages())
277 packages.update(getRuntimeVersions())
278 packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
279 return cls(packages)
281 @classmethod
282 def read(cls, filename):
283 """Read packages from filename.
285 Parameters
286 ----------
287 filename : `str`
288 Filename from which to read.
290 Returns
291 -------
292 packages : `Packages`
293 """
294 with open(filename, "rb") as ff:
295 return pickle.load(ff)
297 def write(self, filename):
298 """Write to file.
300 Parameters
301 ----------
302 filename : `str`
303 Filename to which to write.
304 """
305 with open(filename, "wb") as ff:
306 pickle.dump(self, ff)
308 def __len__(self):
309 return len(self._packages)
311 def __str__(self):
312 ss = "%s({\n" % self.__class__.__name__
313 # Sort alphabetically by module name, for convenience in reading
314 ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
315 prod in sorted(self._names))
316 ss += ",\n})"
317 return ss
319 def __repr__(self):
320 return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
322 def __contains__(self, pkg):
323 return pkg in self._packages
325 def __iter__(self):
326 return iter(self._packages)
328 def update(self, other):
329 """Update packages with contents of another set of packages.
331 Parameters
332 ----------
333 other : `Packages`
334 Other packages to merge with self.
336 Notes
337 -----
338 No check is made to see if we're clobbering anything.
339 """
340 self._packages.update(other._packages)
341 self._names.update(other._names)
343 def extra(self, other):
344 """Get packages in self but not in another `Packages` object.
346 Parameters
347 ----------
348 other : `Packages`
349 Other packages to compare against.
351 Returns
352 -------
353 extra : `dict`
354 Extra packages. Keys (type `str`) are package names; values
355 (type `str`) are their versions.
356 """
357 return {pkg: self._packages[pkg] for pkg in self._names - other._names}
359 def missing(self, other):
360 """Get packages in another `Packages` object but missing from self.
362 Parameters
363 ----------
364 other : `Packages`
365 Other packages to compare against.
367 Returns
368 -------
369 missing : `dict`
370 Missing packages. Keys (type `str`) are package names; values
371 (type `str`) are their versions.
372 """
373 return {pkg: other._packages[pkg] for pkg in other._names - self._names}
375 def difference(self, other):
376 """Get packages in symmetric difference of self and another `Packages`
377 object.
379 Parameters
380 ----------
381 other : `Packages`
382 Other packages to compare against.
384 Returns
385 -------
386 difference : `dict`
387 Packages in symmetric difference. Keys (type `str`) are package
388 names; values (type `str`) are their versions.
389 """
390 return {pkg: (self._packages[pkg], other._packages[pkg]) for
391 pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}