22 Determine which packages are being used in the system and their versions
30 import pickle
as pickle
32 from collections.abc
import Mapping
34 from .versions
import getRuntimeVersions
36 log = logging.getLogger(__name__)
38 __all__ = [
"getVersionFromPythonModule",
"getPythonPackages",
"getEnvironmentPackages",
"Packages"]
42 BUILDTIME = set([
"boost",
"eigen",
"tmv"])
46 PYTHON = set([
"galsim"])
50 ENVIRONMENT = set([
"astrometry_net",
"astrometry_net_data",
"minuit2",
"xpa"])
54 """Determine the version of a python module.
59 Module for which to get version.
68 Raised if __version__ attribute is not set.
72 We supplement the version with information from the
73 ``__dependency_versions__`` (a specific variable set by LSST's
74 `~lsst.sconsUtils` at build time) only for packages that are typically
75 used only at build-time.
77 version = module.__version__
78 if hasattr(module,
"__dependency_versions__"):
80 deps = module.__dependency_versions__
81 buildtime = BUILDTIME & set(deps.keys())
83 version +=
" with " +
" ".join(
"%s=%s" % (pkg, deps[pkg])
84 for pkg
in sorted(buildtime))
89 """Get imported python packages and their versions.
94 Keys (type `str`) are package names; values (type `str`) are their
99 We wade through `sys.modules` and attempt to determine the version for each
100 module. Note, therefore, that we can only report on modules that have
101 *already* been imported.
103 We don't include any module for which we cannot determine a version.
106 for module
in PYTHON:
108 importlib.import_module(module)
112 packages = {
"python": sys.version}
114 moduleNames = list(sys.modules.keys())
115 for name
in moduleNames:
116 module = sys.modules[name]
124 for ending
in (
".version",
"._version"):
125 if name.endswith(ending):
126 name = name[:-len(ending)]
128 assert ver == packages[name]
129 elif name
in packages:
130 assert ver == packages[name]
136 name = name.replace(
"lsst.",
"").replace(
".",
"_")
147 """Get products and their versions from the environment.
152 Keys (type `str`) are product names; values (type `str`) are their
157 We use EUPS to determine the version of certain products (those that don't
158 provide a means to determine the version any other way) and to check if
159 uninstalled packages are being used. We only report the product/version
163 from eups
import Eups
164 from eups.Product
import Product
166 log.warning(
"Unable to import eups, so cannot determine package versions from environment")
173 products = _eups.findProducts(tags=[
"setup"])
177 packages = {prod.name: prod.version
for prod
in products
if prod
in ENVIRONMENT}
183 for prod
in products:
184 if not prod.version.startswith(Product.LocalVersionPrefix):
188 gitDir = os.path.join(prod.dir,
".git")
189 if os.path.exists(gitDir):
191 revCmd = [
"git",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"rev-parse",
"HEAD"]
192 diffCmd = [
"git",
"--no-pager",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"diff",
195 rev = subprocess.check_output(revCmd).decode().strip()
196 diff = subprocess.check_output(diffCmd)
202 ver +=
"+" + hashlib.md5(diff).hexdigest()
206 packages[prod.name] = ver
211 """A table of packages and their versions.
213 There are a few different types of packages, and their versions are collected
216 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
217 interrogating the dynamic library
218 2. Python modules (e.g., afw, numpy; galsim is also in this group even though
219 we only use it through the library, because no version information is
220 currently provided through the library): we get their version from the
221 ``__version__`` module variable. Note that this means that we're only aware
222 of modules that have already been imported.
223 3. Other packages provide no run-time accessible version information (e.g.,
224 astrometry_net): we get their version from interrogating the environment.
225 Currently, that means EUPS; if EUPS is replaced or dropped then we'll need
226 to consider an alternative means of getting this version information.
227 4. Local versions of packages (a non-installed EUPS package, selected with
228 ``setup -r /path/to/package``): we identify these through the environment
229 (EUPS again) and use as a version the path supplemented with the ``git``
230 SHA and, if the git repo isn't clean, an MD5 of the diff.
232 These package versions are collected and stored in a Packages object, which
233 provides useful comparison and persistence features.
237 .. code-block:: python
239 from lsst.base import Packages
240 pkgs = Packages.fromSystem()
241 print("Current packages:", pkgs)
242 old = Packages.read("/path/to/packages.pickle")
243 print("Old packages:", old)
244 print("Missing packages compared to before:", pkgs.missing(old))
245 print("Extra packages compared to before:", pkgs.extra(old))
246 print("Different packages: ", pkgs.difference(old))
247 old.update(pkgs) # Include any new packages in the old
248 old.write("/path/to/packages.pickle")
253 A mapping {package: version} where both keys and values are type `str`.
257 This is essentially a wrapper around a dict with some conveniences.
260 formats = {
".pkl":
"pickle",
265 assert isinstance(packages, Mapping)
267 self.
_names = set(packages.keys())
271 """Construct a `Packages` by examining the system.
273 Determine packages by examining python's `sys.modules`, runtime
278 packages : `Packages`
288 """Construct the object from a byte representation.
293 The serialized form of this object in bytes.
295 The format of those bytes. Can be ``yaml`` or ``pickle``.
297 if format ==
"pickle":
298 new = pickle.loads(data)
299 elif format ==
"yaml":
300 new = yaml.load(data, Loader=yaml.SafeLoader)
302 raise ValueError(f
"Unexpected serialization format given: {format}")
303 if not isinstance(new, cls):
304 raise TypeError(f
"Extracted object of class '{type(new)}' but expected '{cls}'")
309 """Read packages from filename.
314 Filename from which to read. The format is determined from the
315 file extension. Currently support ``.pickle``, ``.pkl``
320 packages : `Packages`
322 _, ext = os.path.splitext(filename)
324 raise ValueError(f
"Format from {ext} extension in file {filename} not recognized")
325 with open(filename,
"rb")
as ff:
332 """Convert the object to a serialized bytes form using the
338 Format to use when serializing. Can be ``yaml`` or ``pickle``.
343 Byte string representing the serialized object.
345 if format ==
"pickle":
346 return pickle.dumps(self)
347 elif format ==
"yaml":
348 return yaml.dump(self).encode(
"utf-8")
350 raise ValueError(f
"Unexpected serialization format requested: {format}")
358 Filename to which to write. The format of the data file
359 is determined from the file extension. Currently supports
360 ``.pickle`` and ``.yaml``
362 _, ext = os.path.splitext(filename)
364 raise ValueError(f
"Format from {ext} extension in file {filename} not recognized")
365 with open(filename,
"wb")
as ff:
374 ss =
"%s({\n" % self.__class__.__name__
376 ss +=
",\n".join(
"%s: %s" % (repr(prod), repr(self.
_packages[prod]))
for
377 prod
in sorted(self.
_names))
382 return "%s(%s)" % (self.__class__.__name__, repr(self.
_packages))
391 if not isinstance(other, type(self)):
397 """Update packages with contents of another set of packages.
402 Other packages to merge with self.
406 No check is made to see if we're clobbering anything.
412 """Get packages in self but not in another `Packages` object.
417 Other packages to compare against.
422 Extra packages. Keys (type `str`) are package names; values
423 (type `str`) are their versions.
425 return {pkg: self.
_packages[pkg]
for pkg
in self.
_names - other._names}
428 """Get packages in another `Packages` object but missing from self.
433 Other packages to compare against.
438 Missing packages. Keys (type `str`) are package names; values
439 (type `str`) are their versions.
441 return {pkg: other._packages[pkg]
for pkg
in other._names - self.
_names}
444 """Get packages in symmetric difference of self and another `Packages`
450 Other packages to compare against.
455 Packages in symmetric difference. Keys (type `str`) are package
456 names; values (type `str`) are their versions.
458 return {pkg: (self.
_packages[pkg], other._packages[pkg])
for
459 pkg
in self.
_names & other._names
if self.
_packages[pkg] != other._packages[pkg]}
465 """Represent Packages as a simple dict"""
466 return dumper.represent_mapping(
"lsst.base.Packages", data._packages,
470 yaml.add_representer(Packages, pkg_representer)
474 yield Packages(loader.construct_mapping(node, deep=
True))
477 for loader
in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
478 yaml.add_constructor(
"lsst.base.Packages", pkg_constructor, Loader=loader)