Coverage for python/lsst/base/packages.py : 21%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""
22Determine which packages are being used in the system and their versions
23"""
24import os
25import sys
26import hashlib
27import importlib
28import subprocess
29import logging
30import pickle as pickle
31import yaml
32from collections.abc import Mapping
34from .versions import getRuntimeVersions
36log = logging.getLogger(__name__)
38__all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
41# Packages used at build-time (e.g., header-only)
42BUILDTIME = set(["boost", "eigen", "tmv"])
44# Python modules to attempt to load so we can try to get the version
45# We do this because the version only appears to be available from python, but we use the library
46PYTHON = set(["galsim"])
48# Packages that don't seem to have a mechanism for reporting the runtime version
49# We need to guess the version from the environment
50ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
53def getVersionFromPythonModule(module):
54 """Determine the version of a python module.
56 Parameters
57 ----------
58 module : `module`
59 Module for which to get version.
61 Returns
62 -------
63 version : `str`
65 Raises
66 ------
67 AttributeError
68 Raised if __version__ attribute is not set.
70 Notes
71 -----
72 We supplement the version with information from the
73 ``__dependency_versions__`` (a specific variable set by LSST's
74 `~lsst.sconsUtils` at build time) only for packages that are typically
75 used only at build-time.
76 """
77 version = module.__version__
78 if hasattr(module, "__dependency_versions__"):
79 # Add build-time dependencies
80 deps = module.__dependency_versions__
81 buildtime = BUILDTIME & set(deps.keys())
82 if buildtime:
83 version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
84 for pkg in sorted(buildtime))
85 return version
88def getPythonPackages():
89 """Get imported python packages and their versions.
91 Returns
92 -------
93 packages : `dict`
94 Keys (type `str`) are package names; values (type `str`) are their
95 versions.
97 Notes
98 -----
99 We wade through `sys.modules` and attempt to determine the version for each
100 module. Note, therefore, that we can only report on modules that have
101 *already* been imported.
103 We don't include any module for which we cannot determine a version.
104 """
105 # Attempt to import libraries that only report their version in python
106 for module in PYTHON:
107 try:
108 importlib.import_module(module)
109 except Exception:
110 pass # It's not available, so don't care
112 packages = {"python": sys.version}
113 # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
114 moduleNames = list(sys.modules.keys())
115 for name in moduleNames:
116 module = sys.modules[name]
117 try:
118 ver = getVersionFromPythonModule(module)
119 except Exception:
120 continue # Can't get a version from it, don't care
122 # Remove "foo.bar.version" in favor of "foo.bar"
123 # This prevents duplication when the __init__.py includes "from .version import *"
124 for ending in (".version", "._version"):
125 if name.endswith(ending):
126 name = name[:-len(ending)]
127 if name in packages:
128 assert ver == packages[name]
129 elif name in packages:
130 assert ver == packages[name]
132 # Use LSST package names instead of python module names
133 # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
134 # versions if the environment reveals that we're not using the packages as-built.
135 if "lsst" in name:
136 name = name.replace("lsst.", "").replace(".", "_")
138 packages[name] = ver
140 return packages
143_eups = None # Singleton Eups object
146def getEnvironmentPackages():
147 """Get products and their versions from the environment.
149 Returns
150 -------
151 packages : `dict`
152 Keys (type `str`) are product names; values (type `str`) are their
153 versions.
155 Notes
156 -----
157 We use EUPS to determine the version of certain products (those that don't
158 provide a means to determine the version any other way) and to check if
159 uninstalled packages are being used. We only report the product/version
160 for these packages.
161 """
162 try:
163 from eups import Eups
164 from eups.Product import Product
165 except ImportError:
166 log.warning("Unable to import eups, so cannot determine package versions from environment")
167 return {}
169 # Cache eups object since creating it can take a while
170 global _eups
171 if not _eups:
172 _eups = Eups()
173 products = _eups.findProducts(tags=["setup"])
175 # Get versions for things we can't determine via runtime mechanisms
176 # XXX Should we just grab everything we can, rather than just a predetermined set?
177 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
179 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
180 # code, so the version could be different than what's being reported by the runtime environment (because
181 # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
182 # probably doesn't check to see if the repo is clean).
183 for prod in products:
184 if not prod.version.startswith(Product.LocalVersionPrefix):
185 continue
186 ver = prod.version
188 gitDir = os.path.join(prod.dir, ".git")
189 if os.path.exists(gitDir):
190 # get the git revision and an indication if the working copy is clean
191 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
192 diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
193 "--patch"]
194 try:
195 rev = subprocess.check_output(revCmd).decode().strip()
196 diff = subprocess.check_output(diffCmd)
197 except Exception:
198 ver += "@GIT_ERROR"
199 else:
200 ver += "@" + rev
201 if diff:
202 ver += "+" + hashlib.md5(diff).hexdigest()
203 else:
204 ver += "@NO_GIT"
206 packages[prod.name] = ver
207 return packages
210class Packages:
211 """A table of packages and their versions.
213 There are a few different types of packages, and their versions are collected
214 in different ways:
216 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
217 interrogating the dynamic library
218 2. Python modules (e.g., afw, numpy; galsim is also in this group even though
219 we only use it through the library, because no version information is
220 currently provided through the library): we get their version from the
221 ``__version__`` module variable. Note that this means that we're only aware
222 of modules that have already been imported.
223 3. Other packages provide no run-time accessible version information (e.g.,
224 astrometry_net): we get their version from interrogating the environment.
225 Currently, that means EUPS; if EUPS is replaced or dropped then we'll need
226 to consider an alternative means of getting this version information.
227 4. Local versions of packages (a non-installed EUPS package, selected with
228 ``setup -r /path/to/package``): we identify these through the environment
229 (EUPS again) and use as a version the path supplemented with the ``git``
230 SHA and, if the git repo isn't clean, an MD5 of the diff.
232 These package versions are collected and stored in a Packages object, which
233 provides useful comparison and persistence features.
235 Example usage:
237 .. code-block:: python
239 from lsst.base import Packages
240 pkgs = Packages.fromSystem()
241 print("Current packages:", pkgs)
242 old = Packages.read("/path/to/packages.pickle")
243 print("Old packages:", old)
244 print("Missing packages compared to before:", pkgs.missing(old))
245 print("Extra packages compared to before:", pkgs.extra(old))
246 print("Different packages: ", pkgs.difference(old))
247 old.update(pkgs) # Include any new packages in the old
248 old.write("/path/to/packages.pickle")
250 Parameters
251 ----------
252 packages : `dict`
253 A mapping {package: version} where both keys and values are type `str`.
255 Notes
256 -----
257 This is essentially a wrapper around a dict with some conveniences.
258 """
260 formats = {".pkl": "pickle",
261 ".pickle": "pickle",
262 ".yaml": "yaml"}
264 def __init__(self, packages):
265 assert isinstance(packages, Mapping)
266 self._packages = packages
267 self._names = set(packages.keys())
269 @classmethod
270 def fromSystem(cls):
271 """Construct a `Packages` by examining the system.
273 Determine packages by examining python's `sys.modules`, runtime
274 libraries and EUPS.
276 Returns
277 -------
278 packages : `Packages`
279 """
280 packages = {}
281 packages.update(getPythonPackages())
282 packages.update(getRuntimeVersions())
283 packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
284 return cls(packages)
286 @classmethod
287 def fromBytes(cls, data, format):
288 """Construct the object from a byte representation.
290 Parameters
291 ----------
292 data : `bytes`
293 The serialized form of this object in bytes.
294 format : `str`
295 The format of those bytes. Can be ``yaml`` or ``pickle``.
296 """
297 if format == "pickle":
298 new = pickle.loads(data)
299 elif format == "yaml":
300 new = yaml.load(data, Loader=yaml.SafeLoader)
301 else:
302 raise ValueError(f"Unexpected serialization format given: {format}")
303 if not isinstance(new, cls):
304 raise TypeError(f"Extracted object of class '{type(new)}' but expected '{cls}'")
305 return new
307 @classmethod
308 def read(cls, filename):
309 """Read packages from filename.
311 Parameters
312 ----------
313 filename : `str`
314 Filename from which to read. The format is determined from the
315 file extension. Currently support ``.pickle``, ``.pkl``
316 and ``.yaml``.
318 Returns
319 -------
320 packages : `Packages`
321 """
322 _, ext = os.path.splitext(filename)
323 if ext not in cls.formats:
324 raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
325 with open(filename, "rb") as ff:
326 # We assume that these classes are tiny so there is no
327 # substantive memory impact by reading the entire file up front
328 data = ff.read()
329 return cls.fromBytes(data, cls.formats[ext])
331 def toBytes(self, format):
332 """Convert the object to a serialized bytes form using the
333 specified format.
335 Parameters
336 ----------
337 format : `str`
338 Format to use when serializing. Can be ``yaml`` or ``pickle``.
340 Returns
341 -------
342 data : `bytes`
343 Byte string representing the serialized object.
344 """
345 if format == "pickle":
346 return pickle.dumps(self)
347 elif format == "yaml":
348 return yaml.dump(self).encode("utf-8")
349 else:
350 raise ValueError(f"Unexpected serialization format requested: {format}")
352 def write(self, filename):
353 """Write to file.
355 Parameters
356 ----------
357 filename : `str`
358 Filename to which to write. The format of the data file
359 is determined from the file extension. Currently supports
360 ``.pickle`` and ``.yaml``
361 """
362 _, ext = os.path.splitext(filename)
363 if ext not in self.formats:
364 raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
365 with open(filename, "wb") as ff:
366 # Assumes that the bytes serialization of this object is
367 # relatively small.
368 ff.write(self.toBytes(self.formats[ext]))
370 def __len__(self):
371 return len(self._packages)
373 def __str__(self):
374 ss = "%s({\n" % self.__class__.__name__
375 # Sort alphabetically by module name, for convenience in reading
376 ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
377 prod in sorted(self._names))
378 ss += ",\n})"
379 return ss
381 def __repr__(self):
382 return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
384 def __contains__(self, pkg):
385 return pkg in self._packages
387 def __iter__(self):
388 return iter(self._packages)
390 def __eq__(self, other):
391 if not isinstance(other, type(self)):
392 return False
394 return self._packages == other._packages
396 def update(self, other):
397 """Update packages with contents of another set of packages.
399 Parameters
400 ----------
401 other : `Packages`
402 Other packages to merge with self.
404 Notes
405 -----
406 No check is made to see if we're clobbering anything.
407 """
408 self._packages.update(other._packages)
409 self._names.update(other._names)
411 def extra(self, other):
412 """Get packages in self but not in another `Packages` object.
414 Parameters
415 ----------
416 other : `Packages`
417 Other packages to compare against.
419 Returns
420 -------
421 extra : `dict`
422 Extra packages. Keys (type `str`) are package names; values
423 (type `str`) are their versions.
424 """
425 return {pkg: self._packages[pkg] for pkg in self._names - other._names}
427 def missing(self, other):
428 """Get packages in another `Packages` object but missing from self.
430 Parameters
431 ----------
432 other : `Packages`
433 Other packages to compare against.
435 Returns
436 -------
437 missing : `dict`
438 Missing packages. Keys (type `str`) are package names; values
439 (type `str`) are their versions.
440 """
441 return {pkg: other._packages[pkg] for pkg in other._names - self._names}
443 def difference(self, other):
444 """Get packages in symmetric difference of self and another `Packages`
445 object.
447 Parameters
448 ----------
449 other : `Packages`
450 Other packages to compare against.
452 Returns
453 -------
454 difference : `dict`
455 Packages in symmetric difference. Keys (type `str`) are package
456 names; values (type `str`) are their versions.
457 """
458 return {pkg: (self._packages[pkg], other._packages[pkg]) for
459 pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}
462# Register YAML representers
464def pkg_representer(dumper, data):
465 """Represent Packages as a simple dict"""
466 return dumper.represent_mapping("lsst.base.Packages", data._packages,
467 flow_style=None)
470yaml.add_representer(Packages, pkg_representer)
473def pkg_constructor(loader, node):
474 yield Packages(loader.construct_mapping(node, deep=True))
477for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
478 yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)