Coverage for python/lsst/utils/packages.py: 86%
249 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-07 22:32 +0000
1# This file is part of utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11#
12"""Determine which packages are being used in the system and their versions."""
14from __future__ import annotations
16import contextlib
17import hashlib
18import importlib
19import io
20import json
21import logging
22import os
23import pickle
24import re
25import subprocess
26import sys
27import types
28from collections.abc import Mapping
29from functools import cache, lru_cache
30from importlib.metadata import packages_distributions
31from typing import Any, ClassVar
33import yaml
35log = logging.getLogger(__name__)
37__all__ = [
38 "Packages",
39 "getAllPythonDistributions",
40 "getCondaPackages",
41 "getEnvironmentPackages",
42 "getPythonPackages",
43 "getVersionFromPythonModule",
44]
47# Packages used at build-time (e.g., header-only)
48BUILDTIME = {"boost", "eigen", "tmv"}
50# Python modules to attempt to load so we can try to get the version
51# We do this because the version only appears to be available from python,
52# but we use the library
53PYTHON: set[str] = set()
55SPECIAL_NAMESPACES = {"lsst"}
57# Packages that don't seem to have a mechanism for reporting the runtime
58# version. We need to guess the version from the environment
59ENVIRONMENT = {"astrometry_net", "astrometry_net_data", "minuit2", "xpa"}
61try:
62 # Python 3.10 includes a list of standard library modules.
63 # These will all have the same version number as Python itself.
64 _STDLIB = sys.stdlib_module_names
65except AttributeError:
66 _STDLIB = frozenset()
69def getVersionFromPythonModule(module: types.ModuleType) -> str:
70 """Determine the version of a python module.
72 Parameters
73 ----------
74 module : `~types.ModuleType`
75 Module for which to get version.
77 Returns
78 -------
79 version : `str`
80 The version of the python module.
82 Raises
83 ------
84 AttributeError
85 Raised if ``__version__`` attribute is not set.
87 Notes
88 -----
89 We supplement the version with information from the
90 ``__dependency_versions__`` (a specific variable set by LSST's
91 `~lsst.sconsUtils` at build time) only for packages that are typically
92 used only at build-time.
93 """
94 version = module.__version__
95 if hasattr(module, "__dependency_versions__"):
96 # Add build-time dependencies
97 deps = module.__dependency_versions__
98 buildtime = BUILDTIME & set(deps.keys())
99 if buildtime:
100 version += " with " + " ".join(f"{pkg}={deps[pkg]}" for pkg in sorted(buildtime))
101 return str(version)
104@cache
105def getAllPythonDistributions() -> dict[str, str]:
106 """Get the versions for all Python distributions that are installed.
108 Returns
109 -------
110 packages : `dict` [ `str`, `str` ]
111 Keys are distribution names; values are their versions.
112 Unlike `getPythonPackages` this function will not include
113 standard library packages defined in `sys.stdlib_module_names` but
114 will include a special ``python`` key reporting the Python version.
116 Notes
117 -----
118 If this function is called a second time an identical result will be
119 returned even if a new distribution has been installed.
120 """
121 packages = {"python": sys.version}
123 for dist in importlib.metadata.distributions():
124 # Keep the first version seen when a distribution
125 # name appears more than once.
126 packages.setdefault(dist.name, dist.version)
127 return _mangle_lsst_package_names(packages)
130def getPythonPackages() -> dict[str, str]:
131 """Get imported python packages and their versions.
133 Returns
134 -------
135 packages : `dict` [ `str`, `str` ]
136 Keys are package names; values are their versions.
138 Notes
139 -----
140 We wade through `sys.modules` and attempt to determine the version for each
141 module. Note, therefore, that we can only report on modules that have
142 *already* been imported.
144 Python standard library packages are not included in the report. A
145 ``python`` key is inserted that records the Python version.
147 We don't include any module for which we cannot determine a version.
149 Whilst distribution names are used to determine package versions, the
150 key returned for the package version is the package name that was imported.
151 This means that ``yaml`` will appear as the version key even though the
152 distribution would be called ``PyYAML``.
153 """
154 # Attempt to import libraries that only report their version in python
155 for module_name in PYTHON: 155 ↛ 157line 155 didn't jump to line 157 because the loop on line 155 never started
156 # If it's not available we continue.
157 with contextlib.suppress(Exception):
158 importlib.import_module(module_name)
160 package_dist = packages_distributions()
162 packages = {"python": sys.version}
164 # Not iterating with sys.modules.iteritems() because it's not atomic and
165 # subject to race conditions
166 module_names = sorted(sys.modules.keys())
168 # Use knowledge of package hierarchy to find the versions rather than
169 # using each name independently. Group all the module names into the
170 # hierarchy, splitting on dot, and skipping any component that starts
171 # with an underscore.
173 # Sorting the module names gives us:
174 # lsst
175 # lsst.afw
176 # lsst.afw.cameraGeom
177 # ...
178 # lsst.daf
179 # lsst.daf.butler
180 #
181 # and so we can use knowledge of the previous version to inform whether
182 # we need to look at the subsequent line.
183 n_versions = 0
184 n_checked = 0
185 previous = ""
186 for name in module_names:
187 if name.startswith("_") or "._" in name:
188 # Refers to a private module so we can ignore it and assume
189 # version has been lifted into parent or, if top level, not
190 # relevant for versioning. This applies also to standard library
191 # packages such as _abc and __future__.
192 continue
194 if name.startswith(previous + ".") and previous in packages:
195 # Already have this version. Use the same previous name
196 # for the line after this.
197 continue
199 # Find the namespace which we need to use package_dist.
200 namespace = name.split(".")[0]
202 if namespace in _STDLIB:
203 # If this is an import from the standard library, skip it.
204 # Standard library names only refer to top-level namespace
205 # so "importlib" appears but "importlib.metadata" does not.
206 previous = name
207 continue
209 # package_dist is a mapping from import namespace to distribution
210 # package names. This may be a one-to-many mapping due to namespace
211 # packages. Note that package_dist does not know about editable
212 # installs or eups installs via path manipulation.
213 if namespace in package_dist:
214 dist_names = package_dist[namespace]
215 else:
216 dist_names = [name]
218 ver = _get_python_package_version(name, namespace, dist_names, packages)
220 n_checked += 1
221 if ver is not None:
222 n_versions += 1
223 previous = name
225 log.debug(
226 "Given %d modules but checked %d in hierarchy and found versions for %d",
227 len(module_names),
228 n_checked,
229 n_versions,
230 )
232 return _mangle_lsst_package_names(packages)
235def _mangle_lsst_package_names(packages: dict[str, str]) -> dict[str, str]:
236 for name in list(packages.keys()):
237 # Use LSST package names instead of python module names
238 # This matches the names we get from the environment (i.e., EUPS)
239 # so we can clobber these build-time versions if the environment
240 # reveals that we're not using the packages as-built.
241 if name.startswith("lsst."):
242 sep = "."
243 elif name.startswith("lsst-"):
244 sep = "-"
245 else:
246 continue
247 new_name = name.replace(f"lsst{sep}", "").replace(sep, "_")
248 packages[new_name] = packages[name]
249 del packages[name]
251 return packages
254def _get_python_package_version(
255 name: str, namespace: str, dist_names: list[str], packages: dict[str, str]
256) -> str | None:
257 """Given a package or module name, try to determine the version.
259 Parameters
260 ----------
261 name : `str`
262 The imported name of the package or module to try.
263 namespace : `str`
264 The namespace of the package or module.
265 dist_names : `list` [ `str` ]
266 The distribution names of the package or module.
267 packages : `dict` [ `str`, `str` ]
268 A dictionary mapping a name to a version. Modified in place.
269 The key used might not match exactly the given key.
271 Returns
272 -------
273 ver : `str` or `None`
274 The version string stored in ``packages``. Nothing is stored if the
275 value here is `None`.
276 """
277 # We have certain special namespaces that are used via eups that
278 # need to be enumerated here.
279 if len(dist_names) > 1 or namespace in SPECIAL_NAMESPACES:
280 # Split the name into parts.
281 name_parts = re.split("[._-]", name)
283 found = False
284 for dist_name in dist_names:
285 # It should be impossible for this to happen but it has happened
286 # so check for it.
287 if dist_name is None: 287 ↛ 288line 287 didn't jump to line 288 because the condition on line 287 was never true
288 continue # type: ignore
289 dist_name_parts = re.split("[._-]", dist_name)
291 # Check if the components start with the namespace; this is
292 # needed because (at least) lsst.ts packages do not use
293 # ``lsst`` in the package name.
294 if dist_name_parts[0] != namespace:
295 dist_name_parts.insert(0, namespace)
297 if dist_name_parts == name_parts:
298 found = True
299 break
301 if not found:
302 # This fallback case occurs when (a) we are testing the overall
303 # namespace (e.g. "lsst" or "sphinxcontrib") and the code below
304 # will return None; or (b) for eups-installed and other
305 # "editable installations" that are not registered as part
306 # of importlib.packages_distributions().
307 dist_name = name
308 else:
309 dist_name = dist_names[0]
311 try:
312 # This is the Python standard way to find a package version.
313 # It can be slow.
314 ver = importlib.metadata.version(dist_name)
315 except Exception:
316 # Fall back to using the module itself. There is no guarantee
317 # that "a" exists for module "a.b" so if hierarchy has been expanded
318 # this might fail. Check first.
319 if name not in sys.modules: 319 ↛ 320line 319 didn't jump to line 320 because the condition on line 319 was never true
320 return None
321 module = sys.modules[name]
322 try:
323 ver = getVersionFromPythonModule(module)
324 except Exception:
325 return None # Can't get a version from it, don't care
327 # Update the package information.
328 if ver is not None: 328 ↛ 331line 328 didn't jump to line 331 because the condition on line 328 was always true
329 packages[name] = ver
331 return ver
334_eups: Any | None = None # Singleton Eups object
337@lru_cache(maxsize=2)
338def getEnvironmentPackages(include_all: bool = False) -> dict[str, str]:
339 """Get products and their versions from the environment.
341 Parameters
342 ----------
343 include_all : `bool`
344 If `False` only returns locally-setup packages. If `True` all set
345 up packages are returned with a version that includes any associated
346 non-current tags.
348 Returns
349 -------
350 packages : `dict`
351 Keys (type `str`) are product names; values (type `str`) are their
352 versions.
354 Notes
355 -----
356 We use EUPS to determine the version of certain products (those that don't
357 provide a means to determine the version any other way) and to check if
358 uninstalled packages are being used. We only report the product/version
359 for these packages unless ``include_all`` is `True`.
361 Assumes that no new EUPS packages are set up after this function is
362 called the first time.
363 """
364 try:
365 from eups import Eups
366 from eups.Product import Product
367 except ImportError:
368 log.warning("Unable to import eups, so cannot determine package versions from environment")
369 return {}
371 # Cache eups object since creating it can take a while
372 global _eups
373 if not _eups:
374 _eups = Eups()
375 products = _eups.findProducts(tags=["setup"])
377 # Get versions for things we can't determine via runtime mechanisms
378 # XXX Should we just grab everything we can, rather than just a
379 # predetermined set?
380 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
382 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the
383 # version name indicates uninstalled code, so the version could be
384 # different than what's being reported by the runtime environment (because
385 # we don't tend to run "scons" every time we update some python file,
386 # and even if we did sconsUtils probably doesn't check to see if the repo
387 # is clean).
388 for prod in products:
389 if not prod.version.startswith(Product.LocalVersionPrefix):
390 if include_all:
391 tags = {t for t in prod.tags if t != "current"}
392 tag_msg = " (" + " ".join(sorted(tags)) + ")" if tags else ""
393 packages[prod.name] = prod.version + tag_msg
394 continue
395 ver = prod.version
397 gitDir = os.path.join(prod.dir, ".git")
398 if os.path.exists(gitDir): 398 ↛ 401line 398 didn't jump to line 401 because the condition on line 398 was never true
399 # get the git revision and an indication if the working copy is
400 # clean
401 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
402 diffCmd = [
403 "git",
404 "--no-pager",
405 "--git-dir=" + gitDir,
406 "--work-tree=" + prod.dir,
407 "diff",
408 "--patch",
409 ]
410 try:
411 rev = subprocess.check_output(revCmd).decode().strip()
412 diff = subprocess.check_output(diffCmd)
413 except Exception:
414 ver += "@GIT_ERROR"
415 else:
416 ver += "@" + rev
417 if diff:
418 ver += "+" + hashlib.md5(diff).hexdigest()
419 else:
420 ver += "@NO_GIT"
422 packages[prod.name] = ver
423 return packages
426@lru_cache(maxsize=1)
427def getCondaPackages() -> dict[str, str]:
428 """Get products and their versions from the conda environment.
430 Returns
431 -------
432 packages : `dict`
433 Keys (type `str`) are product names; values (type `str`) are their
434 versions.
436 Notes
437 -----
438 Returns empty result if a conda environment is not in use or can not
439 be queried.
440 """
441 if "CONDA_PREFIX" not in os.environ: 441 ↛ 442line 441 didn't jump to line 442 because the condition on line 441 was never true
442 return {}
444 # conda list is very slow. Ten times faster to scan the directory
445 # directly. This will only find conda packages and not pip installed
446 # packages.
447 meta_path = os.path.join(os.environ["CONDA_PREFIX"], "conda-meta")
449 try:
450 filenames = os.scandir(path=meta_path)
451 except FileNotFoundError:
452 return {}
454 packages = {}
456 for filename in filenames:
457 if not filename.name.endswith(".json"):
458 continue
459 with open(filename) as f:
460 try:
461 data = json.load(f)
462 except ValueError:
463 continue
464 try:
465 packages[data["name"]] = data["version"]
466 except KeyError:
467 continue
469 packages = dict(sorted(packages.items()))
471 # Try to work out the conda environment name and include it as a fake
472 # package. The "obvious" way of running "conda info --json" does give
473 # access to the active_prefix but takes about 2 seconds to run.
474 # As a compromise look for the env name in the path to the python
475 # executable
476 match = re.search(r"/envs/(.*?)/bin/", sys.executable)
477 if match: 477 ↛ 480line 477 didn't jump to line 480 because the condition on line 477 was always true
478 packages["conda_env"] = match.group(1)
480 return packages
483class Packages(dict):
484 """A table of packages and their versions.
486 There are a few different types of packages, and their versions are
487 collected in different ways:
489 1. Installed Conda packages are obtained via the Conda API. Conda is
490 not required.
491 2. Python modules (e.g., afw, numpy; galsim is also in this group even
492 though we only use it through the library, because no version
493 information is currently provided through the library): we get their
494 version from the ``__version__`` module variable. Note that this means
495 that we're only aware of modules that have already been imported.
496 3. Other packages provide no run-time accessible version information (e.g.,
497 astrometry_net): we get their version from interrogating the
498 environment. Currently, that means EUPS; if EUPS is replaced or dropped
499 then we'll need to consider an alternative means of getting this version
500 information.
501 4. Local versions of packages (a non-installed EUPS package, selected with
502 ``setup -r /path/to/package``): we identify these through the
503 environment (EUPS again) and use as a version the path supplemented with
504 the ``git`` SHA and, if the git repo isn't clean, an MD5 of the diff.
506 These package versions are collected and stored in a Packages object, which
507 provides useful comparison and persistence features.
509 Example usage:
511 .. code-block:: python
513 from lsst.utils.packages import Packages
514 pkgs = Packages.fromSystem()
515 print("Current packages:", pkgs)
516 old = Packages.read("/path/to/packages.pickle")
517 print("Old packages:", old)
518 print("Missing packages compared to before:", pkgs.missing(old))
519 print("Extra packages compared to before:", pkgs.extra(old))
520 print("Different packages: ", pkgs.difference(old))
521 old.update(pkgs) # Include any new packages in the old
522 old.write("/path/to/packages.pickle").
524 Notes
525 -----
526 This is a wrapper around a dict with some convenience methods.
527 """
529 formats: ClassVar[dict[str, str]] = {
530 ".pkl": "pickle",
531 ".pickle": "pickle",
532 ".yaml": "yaml",
533 ".json": "json",
534 }
536 def __setstate__(self, state: dict[str, Any]) -> None:
537 # This only seems to be called for old pickle files where
538 # the data was stored in _packages.
539 self.update(state["_packages"])
541 @classmethod
542 def fromSystem(cls, include_all: bool = False) -> Packages:
543 """Construct a `Packages` by examining the system.
545 Determine packages by examining python's installed packages
546 (by default filtered by `sys.modules`) or distributions, conda
547 libraries and EUPS. EUPS packages take precedence over conda and
548 general python packages.
550 Parameters
551 ----------
552 include_all : `bool`, optional
553 If `False`, will only include imported Python packages, installed
554 Conda packages and locally-setup EUPS packages. If `True` all
555 installed Python distributions and conda packages will be reported
556 as well as all EUPS packages that are set up.
558 Returns
559 -------
560 packages : `Packages`
561 All version package information that could be obtained.
563 Note
564 ----
565 The names of Python distributions can differ from the names of the
566 Python packages installed by those distributions. Since ``include_all``
567 set to `True` uses Python distributions and `False` uses Python
568 packages do not expect that the answers are directly comparable.
569 """
570 packages = {}
571 if include_all:
572 packages.update(getAllPythonDistributions())
573 else:
574 packages.update(getPythonPackages())
575 # Conda list always reports all Conda packages.
576 packages.update(getCondaPackages())
577 # Should be last, to override products with LOCAL versions
578 packages.update(getEnvironmentPackages(include_all=include_all))
579 return cls(packages)
581 @classmethod
582 def fromBytes(cls, data: bytes, format: str) -> Packages:
583 """Construct the object from a byte representation.
585 Parameters
586 ----------
587 data : `bytes`
588 The serialized form of this object in bytes.
589 format : `str`
590 The format of those bytes. Can be ``yaml``, ``json``, or
591 ``pickle``.
593 Returns
594 -------
595 packages : `Packages`
596 The package information read from the input data.
597 """
598 if format == "pickle":
599 file = io.BytesIO(data)
600 new = _BackwardsCompatibilityUnpickler(file).load()
601 elif format == "yaml":
602 new = yaml.load(data, Loader=yaml.SafeLoader)
603 elif format == "json":
604 new = cls(json.loads(data))
605 else:
606 raise ValueError(f"Unexpected serialization format given: {format}")
607 if not isinstance(new, cls):
608 raise TypeError(f"Extracted object of class '{type(new)}' but expected '{cls}'")
609 return new
611 @classmethod
612 def read(cls, filename: str) -> Packages:
613 """Read packages from filename.
615 Parameters
616 ----------
617 filename : `str`
618 Filename from which to read. The format is determined from the
619 file extension. Currently support ``.pickle``, ``.pkl``,
620 ``.json``, and ``.yaml``.
622 Returns
623 -------
624 packages : `Packages`
625 The packages information read from the file.
626 """
627 _, ext = os.path.splitext(filename)
628 if ext not in cls.formats:
629 raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
630 with open(filename, "rb") as ff:
631 # We assume that these classes are tiny so there is no
632 # substantive memory impact by reading the entire file up front
633 data = ff.read()
634 return cls.fromBytes(data, cls.formats[ext])
636 def toBytes(self, format: str) -> bytes:
637 """Convert the object to a serialized bytes form using the
638 specified format.
640 Parameters
641 ----------
642 format : `str`
643 Format to use when serializing. Can be ``yaml``, ``json``,
644 or ``pickle``.
646 Returns
647 -------
648 data : `bytes`
649 Byte string representing the serialized object.
650 """
651 if format == "pickle":
652 return pickle.dumps(self)
653 elif format == "yaml":
654 return yaml.dump(self).encode("utf-8")
655 elif format == "json":
656 return json.dumps(self).encode("utf-8")
657 else:
658 raise ValueError(f"Unexpected serialization format requested: {format}")
660 def write(self, filename: str) -> None:
661 """Write to file.
663 Parameters
664 ----------
665 filename : `str`
666 Filename to which to write. The format of the data file
667 is determined from the file extension. Currently supports
668 ``.pickle``, ``.json``, and ``.yaml``.
669 """
670 _, ext = os.path.splitext(filename)
671 if ext not in self.formats:
672 raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
673 with open(filename, "wb") as ff:
674 # Assumes that the bytes serialization of this object is
675 # relatively small.
676 ff.write(self.toBytes(self.formats[ext]))
678 def __str__(self) -> str:
679 ss = self.__class__.__name__ + "({\n"
680 # Sort alphabetically by module name, for convenience in reading
681 ss += ",\n".join(f"{prod!r}:{self[prod]!r}" for prod in sorted(self))
682 ss += ",\n})"
683 return ss
685 def __repr__(self) -> str:
686 # Default repr() does not report the class name.
687 return f"{self.__class__.__name__}({super().__repr__()})"
689 def extra(self, other: Mapping) -> dict[str, str]:
690 """Get packages in self but not in another `Packages` object.
692 Parameters
693 ----------
694 other : `Packages` or `Mapping`
695 Other packages to compare against.
697 Returns
698 -------
699 extra : `dict`
700 Extra packages. Keys (type `str`) are package names; values
701 (type `str`) are their versions.
702 """
703 return {pkg: self[pkg] for pkg in self.keys() - other.keys()}
705 def missing(self, other: Mapping) -> dict[str, str]:
706 """Get packages in another `Packages` object but missing from self.
708 Parameters
709 ----------
710 other : `Packages`
711 Other packages to compare against.
713 Returns
714 -------
715 missing : `dict` [`str`, `str`]
716 Missing packages. Keys (type `str`) are package names; values
717 (type `str`) are their versions.
718 """
719 return {pkg: other[pkg] for pkg in other.keys() - self.keys()}
721 def difference(self, other: Mapping) -> dict[str, tuple[str, str]]:
722 """Get packages in symmetric difference of self and another `Packages`
723 object.
725 Parameters
726 ----------
727 other : `Packages`
728 Other packages to compare against.
730 Returns
731 -------
732 difference : `dict` [`str`, `tuple` [ `str`, `str` ]]
733 Packages in symmetric difference. Keys (type `str`) are package
734 names; values (type `tuple` [ `str`, `str` ]) are their versions.
735 """
736 return {pkg: (self[pkg], other[pkg]) for pkg in self.keys() & other.keys() if self[pkg] != other[pkg]}
739class _BackwardsCompatibilityUnpickler(pickle.Unpickler):
740 """Replacement for the default unpickler.
742 It is required so that users of this API can read pickle files
743 created when the `~lsst.utils.packages.Packages` class was in a different
744 package and known as ``lsst.base.Packages``. If this unpickler is being
745 used then we know for sure that we must return a
746 `~lsst.utils.packages.Packages` instance.
747 """
749 def find_class(self, module: str, name: str) -> type:
750 """Return the class that should be used for unpickling.
752 This is always known to be the class in this package.
754 Parameters
755 ----------
756 module : `str`
757 Ignored.
758 name : `str`
759 Ignored.
761 Returns
762 -------
763 `type` [`Packages`]
764 The Python type to use. Always returns `Packages`.
765 """
766 return Packages
769# Register YAML representers
772def _pkg_representer(dumper: yaml.Dumper, data: Any) -> yaml.MappingNode:
773 """Represent Packages as a simple dict"""
774 return dumper.represent_mapping("lsst.utils.packages.Packages", data, flow_style=None)
777yaml.add_representer(Packages, _pkg_representer)
780def _pkg_constructor(loader: yaml.constructor.SafeConstructor, node: yaml.Node) -> Any:
781 """Convert YAML representation back to Python class."""
782 yield Packages(loader.construct_mapping(node, deep=True)) # type: ignore
785for loader_str in ("Loader", "CLoader", "UnsafeLoader", "SafeLoader", "FullLoader"):
786 loader = getattr(yaml, loader_str, None)
787 if loader is None: 787 ↛ 788line 787 didn't jump to line 788 because the condition on line 787 was never true
788 continue
790 yaml.add_constructor("lsst.utils.packages.Packages", _pkg_constructor, Loader=loader)
792 # Register the old name with YAML.
793 yaml.add_constructor("lsst.base.Packages", _pkg_constructor, Loader=loader)