22Determine which packages are being used in the system and their versions
30import pickle
as pickle
33from functools
import lru_cache
35from .versions
import getRuntimeVersions
37log = logging.getLogger(__name__)
39__all__ = [
"getVersionFromPythonModule",
"getPythonPackages",
"getEnvironmentPackages",
40 "getCondaPackages",
"Packages"]
44BUILDTIME = set([
"boost",
"eigen",
"tmv"])
49PYTHON = set([
"galsim"])
53ENVIRONMENT = set([
"astrometry_net",
"astrometry_net_data",
"minuit2",
"xpa"])
57 """Determine the version of a python module.
62 Module for which to get version.
71 Raised
if __version__ attribute
is not set.
75 We supplement the version
with information
from the
76 ``__dependency_versions__`` (a specific variable set by LSST
's
77 `~lsst.sconsUtils` at build time) only for packages that are typically
78 used only at build-time.
80 version = module.__version__
81 if hasattr(module,
"__dependency_versions__"):
83 deps = module.__dependency_versions__
84 buildtime = BUILDTIME & set(deps.keys())
86 version +=
" with " +
" ".join(
"%s=%s" % (pkg, deps[pkg])
87 for pkg
in sorted(buildtime))
92 """Get imported python packages and their versions.
97 Keys (type `str`) are package names; values (type `str`) are their
102 We wade through `sys.modules` and attempt to determine the version
for each
103 module. Note, therefore, that we can only report on modules that have
104 *already* been imported.
106 We don
't include any module for which we cannot determine a version.
109 for module
in PYTHON:
111 importlib.import_module(module)
115 packages = {
"python": sys.version}
118 moduleNames = list(sys.modules.keys())
119 for name
in moduleNames:
120 module = sys.modules[name]
129 for ending
in (
".version",
"._version"):
130 if name.endswith(ending):
131 name = name[:-len(ending)]
133 assert ver == packages[name]
134 elif name
in packages:
135 assert ver == packages[name]
142 name = name.replace(
"lsst.",
"").replace(
".",
"_")
154 """Get products and their versions from the environment.
159 Keys (type `str`) are product names; values (type `str`) are their
164 We use EUPS to determine the version of certain products (those that don't
165 provide a means to determine the version any other way) and to check
if
166 uninstalled packages are being used. We only report the product/version
170 from eups
import Eups
171 from eups.Product
import Product
173 log.warning(
"Unable to import eups, so cannot determine package versions from environment")
180 products = _eups.findProducts(tags=[
"setup"])
185 packages = {prod.name: prod.version
for prod
in products
if prod
in ENVIRONMENT}
193 for prod
in products:
194 if not prod.version.startswith(Product.LocalVersionPrefix):
198 gitDir = os.path.join(prod.dir,
".git")
199 if os.path.exists(gitDir):
202 revCmd = [
"git",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"rev-parse",
"HEAD"]
203 diffCmd = [
"git",
"--no-pager",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"diff",
206 rev = subprocess.check_output(revCmd).decode().strip()
207 diff = subprocess.check_output(diffCmd)
213 ver +=
"+" + hashlib.md5(diff).hexdigest()
217 packages[prod.name] = ver
223 """Get products and their versions from the conda environment.
228 Keys (type `str`) are product names; values (type `str`) are their
233 Returns empty result if a conda environment
is not in use
or can
not
239 from conda.cli.python_api
import Commands, run_command
244 versions_json = run_command(Commands.LIST,
"--json")
245 packages = {pkg[
"name"]: pkg[
"version"]
for pkg
in json.loads(versions_json[0])}
254 match = re.search(
r"/envs/(.*?)/bin/", sys.executable)
256 packages[
"conda_env"] = match.group(1)
262 """A table of packages and their versions.
264 There are a few different types of packages, and their versions are
265 collected
in different ways:
267 1. Run-time libraries (e.g., cfitsio, fftw): we get their version
from
268 interrogating the dynamic library
269 2. Python modules (e.g., afw, numpy; galsim
is also
in this group even
270 though we only use it through the library, because no version
271 information
is currently provided through the library): we get their
272 version
from the ``__version__`` module variable. Note that this means
273 that we
're only aware of modules that have already been imported.
274 3. Other packages provide no run-time accessible version information (e.g.,
275 astrometry_net): we get their version from interrogating the
276 environment. Currently, that means EUPS;
if EUPS
is replaced
or dropped
277 then we
'll need to consider an alternative means of getting this version
279 4. Local versions of packages (a non-installed EUPS package, selected with
280 ``setup -r /path/to/package``): we identify these through the
281 environment (EUPS again)
and use
as a version the path supplemented
with
282 the ``git`` SHA
and,
if the git repo isn
't clean, an MD5 of the diff.
284 These package versions are collected and stored
in a Packages object, which
285 provides useful comparison
and persistence features.
289 .. code-block:: python
292 pkgs = Packages.fromSystem()
293 print(
"Current packages:", pkgs)
294 old = Packages.read(
"/path/to/packages.pickle")
295 print(
"Old packages:", old)
296 print(
"Missing packages compared to before:", pkgs.missing(old))
297 print(
"Extra packages compared to before:", pkgs.extra(old))
298 print(
"Different packages: ", pkgs.difference(old))
300 old.write(
"/path/to/packages.pickle")
305 A mapping {package: version} where both keys
and values are type `str`.
309 This
is essentially a wrapper around a dict
with some conveniences.
312 formats = {".pkl":
"pickle",
319 self.update(state[
"_packages"])
323 """Construct a `Packages` by examining the system.
325 Determine packages by examining python's `sys.modules`, runtime
330 packages : `Packages`
341 """Construct the object from a byte representation.
346 The serialized form of this object in bytes.
348 The format of those bytes. Can be ``yaml``
or ``pickle``.
350 if format ==
"pickle":
351 new = pickle.loads(data)
352 elif format ==
"yaml":
353 new = yaml.load(data, Loader=yaml.SafeLoader)
355 raise ValueError(f
"Unexpected serialization format given: {format}")
356 if not isinstance(new, cls):
357 raise TypeError(f
"Extracted object of class '{type(new)}' but expected '{cls}'")
362 """Read packages from filename.
367 Filename from which to read. The format
is determined
from the
368 file extension. Currently support ``.pickle``, ``.pkl``
373 packages : `Packages`
375 _, ext = os.path.splitext(filename)
376 if ext
not in cls.
formatsformats:
377 raise ValueError(f
"Format from {ext} extension in file {filename} not recognized")
378 with open(filename,
"rb")
as ff:
385 """Convert the object to a serialized bytes form using the
391 Format to use when serializing. Can be ``yaml`` or ``pickle``.
396 Byte string representing the serialized object.
398 if format ==
"pickle":
399 return pickle.dumps(self)
400 elif format ==
"yaml":
401 return yaml.dump(self).encode(
"utf-8")
403 raise ValueError(f
"Unexpected serialization format requested: {format}")
411 Filename to which to write. The format of the data file
412 is determined
from the file extension. Currently supports
413 ``.pickle``
and ``.yaml``
415 _, ext = os.path.splitext(filename)
416 if ext
not in self.
formatsformats:
417 raise ValueError(f
"Format from {ext} extension in file {filename} not recognized")
418 with open(filename,
"wb")
as ff:
424 ss =
"%s({\n" % self.__class__.__name__
426 ss +=
",\n".join(f
"{prod!r}:{self[prod]!r}" for prod
in sorted(self))
432 return f
"{self.__class__.__name__}({super().__repr__()})"
435 """Get packages in self but not in another `Packages` object.
439 other : `Packages` or `Mapping`
440 Other packages to compare against.
445 Extra packages. Keys (type `str`) are package names; values
446 (type `str`) are their versions.
448 return {pkg: self[pkg]
for pkg
in self.keys() - other.keys()}
451 """Get packages in another `Packages` object but missing from self.
456 Other packages to compare against.
461 Missing packages. Keys (type `str`) are package names; values
462 (type `str`) are their versions.
464 return {pkg: other[pkg]
for pkg
in other.keys() - self.keys()}
467 """Get packages in symmetric difference of self and another `Packages`
473 Other packages to compare against.
478 Packages in symmetric difference. Keys (type `str`) are package
479 names; values (type `str`) are their versions.
481 return {pkg: (self[pkg], other[pkg])
for
482 pkg
in self.keys() & other.keys()
if self[pkg] != other[pkg]}
488 """Represent Packages as a simple dict"""
489 return dumper.represent_mapping(
"lsst.base.Packages", data,
493yaml.add_representer(Packages, pkg_representer)
497 yield Packages(loader.construct_mapping(node, deep=
True))
500for loader
in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
501 yaml.add_constructor(
"lsst.base.Packages", pkg_constructor, Loader=loader)
def fromBytes(cls, data, format)
def __setstate__(self, state)
def write(self, filename)
def difference(self, other)
def toBytes(self, format)
def getEnvironmentPackages()
def pkg_representer(dumper, data)
def getVersionFromPythonModule(module)
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.