22 Determine which packages are being used in the system and their versions
30 import pickle
as pickle
32 from collections.abc
import Mapping
34 from .versions
import getRuntimeVersions
36 log = logging.getLogger(__name__)
38 __all__ = [
"getVersionFromPythonModule",
"getPythonPackages",
"getEnvironmentPackages",
"Packages"]
42 BUILDTIME = set([
"boost",
"eigen",
"tmv"])
46 PYTHON = set([
"galsim"])
50 ENVIRONMENT = set([
"astrometry_net",
"astrometry_net_data",
"minuit2",
"xpa"])
54 """Determine the version of a python module.
59 Module for which to get version.
68 Raised if __version__ attribute is not set.
72 We supplement the version with information from the
73 ``__dependency_versions__`` (a specific variable set by LSST's
74 `~lsst.sconsUtils` at build time) only for packages that are typically
75 used only at build-time.
77 version = module.__version__
78 if hasattr(module,
"__dependency_versions__"):
80 deps = module.__dependency_versions__
81 buildtime = BUILDTIME & set(deps.keys())
83 version +=
" with " +
" ".join(
"%s=%s" % (pkg, deps[pkg])
84 for pkg
in sorted(buildtime))
89 """Get imported python packages and their versions.
94 Keys (type `str`) are package names; values (type `str`) are their
99 We wade through `sys.modules` and attempt to determine the version for each
100 module. Note, therefore, that we can only report on modules that have
101 *already* been imported.
103 We don't include any module for which we cannot determine a version.
106 for module
in PYTHON:
108 importlib.import_module(module)
112 packages = {
"python": sys.version}
114 moduleNames = list(sys.modules.keys())
115 for name
in moduleNames:
116 module = sys.modules[name]
124 for ending
in (
".version",
"._version"):
125 if name.endswith(ending):
126 name = name[:-len(ending)]
128 assert ver == packages[name]
129 elif name
in packages:
130 assert ver == packages[name]
136 name = name.replace(
"lsst.",
"").replace(
".",
"_")
147 """Get products and their versions from the environment.
152 Keys (type `str`) are product names; values (type `str`) are their
157 We use EUPS to determine the version of certain products (those that don't
158 provide a means to determine the version any other way) and to check if
159 uninstalled packages are being used. We only report the product/version
163 from eups
import Eups
164 from eups.Product
import Product
166 log.warning(
"Unable to import eups, so cannot determine package versions from environment")
173 products = _eups.findProducts(tags=[
"setup"])
177 packages = {prod.name: prod.version
for prod
in products
if prod
in ENVIRONMENT}
183 for prod
in products:
184 if not prod.version.startswith(Product.LocalVersionPrefix):
188 gitDir = os.path.join(prod.dir,
".git")
189 if os.path.exists(gitDir):
191 revCmd = [
"git",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"rev-parse",
"HEAD"]
192 diffCmd = [
"git",
"--no-pager",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"diff",
195 rev = subprocess.check_output(revCmd).decode().strip()
196 diff = subprocess.check_output(diffCmd)
202 ver +=
"+" + hashlib.md5(diff).hexdigest()
206 packages[prod.name] = ver
211 """A table of packages and their versions.
213 There are a few different types of packages, and their versions are collected
216 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
217 interrogating the dynamic library
218 2. Python modules (e.g., afw, numpy; galsim is also in this group even though
219 we only use it through the library, because no version information is
220 currently provided through the library): we get their version from the
221 ``__version__`` module variable. Note that this means that we're only aware
222 of modules that have already been imported.
223 3. Other packages provide no run-time accessible version information (e.g.,
224 astrometry_net): we get their version from interrogating the environment.
225 Currently, that means EUPS; if EUPS is replaced or dropped then we'll need
226 to consider an alternative means of getting this version information.
227 4. Local versions of packages (a non-installed EUPS package, selected with
228 ``setup -r /path/to/package``): we identify these through the environment
229 (EUPS again) and use as a version the path supplemented with the ``git``
230 SHA and, if the git repo isn't clean, an MD5 of the diff.
232 These package versions are collected and stored in a Packages object, which
233 provides useful comparison and persistence features.
237 .. code-block:: python
239 from lsst.base import Packages
240 pkgs = Packages.fromSystem()
241 print("Current packages:", pkgs)
242 old = Packages.read("/path/to/packages.pickle")
243 print("Old packages:", old)
244 print("Missing packages compared to before:", pkgs.missing(old))
245 print("Extra packages compared to before:", pkgs.extra(old))
246 print("Different packages: ", pkgs.difference(old))
247 old.update(pkgs) # Include any new packages in the old
248 old.write("/path/to/packages.pickle")
253 A mapping {package: version} where both keys and values are type `str`.
257 This is essentially a wrapper around a dict with some conveniences.
261 assert isinstance(packages, Mapping)
263 self.
_names = set(packages.keys())
267 """Construct a `Packages` by examining the system.
269 Determine packages by examining python's `sys.modules`, runtime
274 packages : `Packages`
284 """Read packages from filename.
289 Filename from which to read. The format is determined from the
290 file extension. Currently support ``.pickle``, ``.pkl``
295 packages : `Packages`
297 _, ext = os.path.splitext(filename)
298 if ext
in (
".pickle",
".pkl"):
299 with open(filename,
"rb")
as ff:
300 return pickle.load(ff)
302 with open(filename,
"r")
as ff:
303 return yaml.load(ff, Loader=yaml.SafeLoader)
305 raise ValueError(f
"Unable to determine how to read file {filename} from extension {ext}")
313 Filename to which to write. The format of the data file
314 is determined from the file extension. Currently supports
315 ``.pickle`` and ``.yaml``
317 _, ext = os.path.splitext(filename)
318 if ext
in (
".pickle",
".pkl"):
319 with open(filename,
"wb")
as ff:
320 pickle.dump(self, ff)
322 with open(filename,
"w")
as ff:
325 raise ValueError(f
"Unexpected file format requested: {ext}")
331 ss =
"%s({\n" % self.__class__.__name__
333 ss +=
",\n".join(
"%s: %s" % (repr(prod), repr(self.
_packages[prod]))
for
334 prod
in sorted(self.
_names))
339 return "%s(%s)" % (self.__class__.__name__, repr(self.
_packages))
348 if not isinstance(other, type(self)):
354 """Update packages with contents of another set of packages.
359 Other packages to merge with self.
363 No check is made to see if we're clobbering anything.
369 """Get packages in self but not in another `Packages` object.
374 Other packages to compare against.
379 Extra packages. Keys (type `str`) are package names; values
380 (type `str`) are their versions.
382 return {pkg: self.
_packages[pkg]
for pkg
in self.
_names - other._names}
385 """Get packages in another `Packages` object but missing from self.
390 Other packages to compare against.
395 Missing packages. Keys (type `str`) are package names; values
396 (type `str`) are their versions.
398 return {pkg: other._packages[pkg]
for pkg
in other._names - self.
_names}
401 """Get packages in symmetric difference of self and another `Packages`
407 Other packages to compare against.
412 Packages in symmetric difference. Keys (type `str`) are package
413 names; values (type `str`) are their versions.
415 return {pkg: (self.
_packages[pkg], other._packages[pkg])
for
416 pkg
in self.
_names & other._names
if self.
_packages[pkg] != other._packages[pkg]}
422 """Represent Packages as a simple dict"""
423 return dumper.represent_mapping(
"lsst.base.Packages", data._packages,
427 yaml.add_representer(Packages, pkg_representer)
431 yield Packages(loader.construct_mapping(node, deep=
True))
434 for loader
in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
435 yaml.add_constructor(
"lsst.base.Packages", pkg_constructor, Loader=loader)