lsst.base ga9b0f51b18+052faf71bd
LSST Data Management Base Package
packages.py
Go to the documentation of this file.
1# This file is part of base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""
22Determine which packages are being used in the system and their versions
23"""
24import os
25import sys
26import hashlib
27import importlib
28import subprocess
29import logging
30import pickle as pickle
31import re
32import yaml
33from collections.abc import Mapping
34from functools import lru_cache
35
36from .versions import getRuntimeVersions
37
38log = logging.getLogger(__name__)
39
40__all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages",
41 "getCondaPackages", "Packages"]
42
43
44# Packages used at build-time (e.g., header-only)
45BUILDTIME = set(["boost", "eigen", "tmv"])
46
47# Python modules to attempt to load so we can try to get the version
48# We do this because the version only appears to be available from python,
49# but we use the library
50PYTHON = set(["galsim"])
51
52# Packages that don't seem to have a mechanism for reporting the runtime
53# version. We need to guess the version from the environment
54ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
55
56
58 """Determine the version of a python module.
59
60 Parameters
61 ----------
62 module : `module`
63 Module for which to get version.
64
65 Returns
66 -------
67 version : `str`
68
69 Raises
70 ------
71 AttributeError
72 Raised if __version__ attribute is not set.
73
74 Notes
75 -----
76 We supplement the version with information from the
77 ``__dependency_versions__`` (a specific variable set by LSST's
78 `~lsst.sconsUtils` at build time) only for packages that are typically
79 used only at build-time.
80 """
81 version = module.__version__
82 if hasattr(module, "__dependency_versions__"):
83 # Add build-time dependencies
84 deps = module.__dependency_versions__
85 buildtime = BUILDTIME & set(deps.keys())
86 if buildtime:
87 version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
88 for pkg in sorted(buildtime))
89 return str(version)
90
91
93 """Get imported python packages and their versions.
94
95 Returns
96 -------
97 packages : `dict`
98 Keys (type `str`) are package names; values (type `str`) are their
99 versions.
100
101 Notes
102 -----
103 We wade through `sys.modules` and attempt to determine the version for each
104 module. Note, therefore, that we can only report on modules that have
105 *already* been imported.
106
107 We don't include any module for which we cannot determine a version.
108 """
109 # Attempt to import libraries that only report their version in python
110 for module in PYTHON:
111 try:
112 importlib.import_module(module)
113 except Exception:
114 pass # It's not available, so don't care
115
116 packages = {"python": sys.version}
117 # Not iterating with sys.modules.iteritems() because it's not atomic and
118 # subject to race conditions
119 moduleNames = list(sys.modules.keys())
120 for name in moduleNames:
121 module = sys.modules[name]
122 try:
123 ver = getVersionFromPythonModule(module)
124 except Exception:
125 continue # Can't get a version from it, don't care
126
127 # Remove "foo.bar.version" in favor of "foo.bar"
128 # This prevents duplication when the __init__.py includes
129 # "from .version import *"
130 for ending in (".version", "._version"):
131 if name.endswith(ending):
132 name = name[:-len(ending)]
133 if name in packages:
134 assert ver == packages[name]
135 elif name in packages:
136 assert ver == packages[name]
137
138 # Use LSST package names instead of python module names
139 # This matches the names we get from the environment (i.e., EUPS)
140 # so we can clobber these build-time versions if the environment
141 # reveals that we're not using the packages as-built.
142 if "lsst" in name:
143 name = name.replace("lsst.", "").replace(".", "_")
144
145 packages[name] = ver
146
147 return packages
148
149
150_eups = None # Singleton Eups object
151
152
153@lru_cache(maxsize=1)
155 """Get products and their versions from the environment.
156
157 Returns
158 -------
159 packages : `dict`
160 Keys (type `str`) are product names; values (type `str`) are their
161 versions.
162
163 Notes
164 -----
165 We use EUPS to determine the version of certain products (those that don't
166 provide a means to determine the version any other way) and to check if
167 uninstalled packages are being used. We only report the product/version
168 for these packages.
169 """
170 try:
171 from eups import Eups
172 from eups.Product import Product
173 except ImportError:
174 log.warning("Unable to import eups, so cannot determine package versions from environment")
175 return {}
176
177 # Cache eups object since creating it can take a while
178 global _eups
179 if not _eups:
180 _eups = Eups()
181 products = _eups.findProducts(tags=["setup"])
182
183 # Get versions for things we can't determine via runtime mechanisms
184 # XXX Should we just grab everything we can, rather than just a
185 # predetermined set?
186 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
187
188 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the
189 # version name indicates uninstalled code, so the version could be
190 # different than what's being reported by the runtime environment (because
191 # we don't tend to run "scons" every time we update some python file,
192 # and even if we did sconsUtils probably doesn't check to see if the repo
193 # is clean).
194 for prod in products:
195 if not prod.version.startswith(Product.LocalVersionPrefix):
196 continue
197 ver = prod.version
198
199 gitDir = os.path.join(prod.dir, ".git")
200 if os.path.exists(gitDir):
201 # get the git revision and an indication if the working copy is
202 # clean
203 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
204 diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
205 "--patch"]
206 try:
207 rev = subprocess.check_output(revCmd).decode().strip()
208 diff = subprocess.check_output(diffCmd)
209 except Exception:
210 ver += "@GIT_ERROR"
211 else:
212 ver += "@" + rev
213 if diff:
214 ver += "+" + hashlib.md5(diff).hexdigest()
215 else:
216 ver += "@NO_GIT"
217
218 packages[prod.name] = ver
219 return packages
220
221
222@lru_cache(maxsize=1)
224 """Get products and their versions from the conda environment.
225
226 Returns
227 -------
228 packages : `dict`
229 Keys (type `str`) are product names; values (type `str`) are their
230 versions.
231
232 Notes
233 -----
234 Returns empty result if a conda environment is not in use or can not
235 be queried.
236 """
237
238 try:
239 import json
240 from conda.cli.python_api import Commands, run_command
241 except ImportError:
242 return {}
243
244 # Get the installed package list
245 versions_json = run_command(Commands.LIST, "--json")
246 packages = {pkg["name"]: pkg["version"] for pkg in json.loads(versions_json[0])}
247
248 # Try to work out the conda environment name and include it as a fake
249 # package. The "obvious" way of running "conda info --json" does give
250 # access to the active_prefix but takes about 2 seconds to run.
251 # The equivalent to the code above would be:
252 # info_json = run_command(Commands.INFO, "--json")
253 # As a comporomise look for the env name in the path to the python
254 # executable
255 match = re.search(r"/envs/(.*?)/bin/", sys.executable)
256 if match:
257 packages["conda_env"] = match.group(1)
258
259 return packages
260
261
263 """A table of packages and their versions.
264
265 There are a few different types of packages, and their versions are
266 collected in different ways:
267
268 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
269 interrogating the dynamic library
270 2. Python modules (e.g., afw, numpy; galsim is also in this group even
271 though we only use it through the library, because no version
272 information is currently provided through the library): we get their
273 version from the ``__version__`` module variable. Note that this means
274 that we're only aware of modules that have already been imported.
275 3. Other packages provide no run-time accessible version information (e.g.,
276 astrometry_net): we get their version from interrogating the
277 environment. Currently, that means EUPS; if EUPS is replaced or dropped
278 then we'll need to consider an alternative means of getting this version
279 information.
280 4. Local versions of packages (a non-installed EUPS package, selected with
281 ``setup -r /path/to/package``): we identify these through the
282 environment (EUPS again) and use as a version the path supplemented with
283 the ``git`` SHA and, if the git repo isn't clean, an MD5 of the diff.
284
285 These package versions are collected and stored in a Packages object, which
286 provides useful comparison and persistence features.
287
288 Example usage:
289
290 .. code-block:: python
291
292 from lsst.base import Packages
293 pkgs = Packages.fromSystem()
294 print("Current packages:", pkgs)
295 old = Packages.read("/path/to/packages.pickle")
296 print("Old packages:", old)
297 print("Missing packages compared to before:", pkgs.missing(old))
298 print("Extra packages compared to before:", pkgs.extra(old))
299 print("Different packages: ", pkgs.difference(old))
300 old.update(pkgs) # Include any new packages in the old
301 old.write("/path/to/packages.pickle")
302
303 Parameters
304 ----------
305 packages : `dict`
306 A mapping {package: version} where both keys and values are type `str`.
307
308 Notes
309 -----
310 This is essentially a wrapper around a dict with some conveniences.
311 """
312
313 formats = {".pkl": "pickle",
314 ".pickle": "pickle",
315 ".yaml": "yaml"}
316
317 def __init__(self, packages):
318 assert isinstance(packages, Mapping)
319 self._packages_packages = packages
320 self._names_names = set(packages.keys())
321
322 @classmethod
323 def fromSystem(cls):
324 """Construct a `Packages` by examining the system.
325
326 Determine packages by examining python's `sys.modules`, runtime
327 libraries and EUPS.
328
329 Returns
330 -------
331 packages : `Packages`
332 """
333 packages = {}
334 packages.update(getPythonPackages())
335 packages.update(getCondaPackages())
336 packages.update(getRuntimeVersions())
337 packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
338 return cls(packages)
339
340 @classmethod
341 def fromBytes(cls, data, format):
342 """Construct the object from a byte representation.
343
344 Parameters
345 ----------
346 data : `bytes`
347 The serialized form of this object in bytes.
348 format : `str`
349 The format of those bytes. Can be ``yaml`` or ``pickle``.
350 """
351 if format == "pickle":
352 new = pickle.loads(data)
353 elif format == "yaml":
354 new = yaml.load(data, Loader=yaml.SafeLoader)
355 else:
356 raise ValueError(f"Unexpected serialization format given: {format}")
357 if not isinstance(new, cls):
358 raise TypeError(f"Extracted object of class '{type(new)}' but expected '{cls}'")
359 return new
360
361 @classmethod
362 def read(cls, filename):
363 """Read packages from filename.
364
365 Parameters
366 ----------
367 filename : `str`
368 Filename from which to read. The format is determined from the
369 file extension. Currently support ``.pickle``, ``.pkl``
370 and ``.yaml``.
371
372 Returns
373 -------
374 packages : `Packages`
375 """
376 _, ext = os.path.splitext(filename)
377 if ext not in cls.formatsformats:
378 raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
379 with open(filename, "rb") as ff:
380 # We assume that these classes are tiny so there is no
381 # substantive memory impact by reading the entire file up front
382 data = ff.read()
383 return cls.fromBytesfromBytes(data, cls.formatsformats[ext])
384
385 def toBytes(self, format):
386 """Convert the object to a serialized bytes form using the
387 specified format.
388
389 Parameters
390 ----------
391 format : `str`
392 Format to use when serializing. Can be ``yaml`` or ``pickle``.
393
394 Returns
395 -------
396 data : `bytes`
397 Byte string representing the serialized object.
398 """
399 if format == "pickle":
400 return pickle.dumps(self)
401 elif format == "yaml":
402 return yaml.dump(self).encode("utf-8")
403 else:
404 raise ValueError(f"Unexpected serialization format requested: {format}")
405
406 def write(self, filename):
407 """Write to file.
408
409 Parameters
410 ----------
411 filename : `str`
412 Filename to which to write. The format of the data file
413 is determined from the file extension. Currently supports
414 ``.pickle`` and ``.yaml``
415 """
416 _, ext = os.path.splitext(filename)
417 if ext not in self.formatsformats:
418 raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
419 with open(filename, "wb") as ff:
420 # Assumes that the bytes serialization of this object is
421 # relatively small.
422 ff.write(self.toBytestoBytes(self.formatsformats[ext]))
423
424 def __len__(self):
425 return len(self._packages_packages)
426
427 def __str__(self):
428 ss = "%s({\n" % self.__class__.__name__
429 # Sort alphabetically by module name, for convenience in reading
430 ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages_packages[prod])) for
431 prod in sorted(self._names_names))
432 ss += ",\n})"
433 return ss
434
435 def __repr__(self):
436 return "%s(%s)" % (self.__class__.__name__, repr(self._packages_packages))
437
438 def __contains__(self, pkg):
439 return pkg in self._packages_packages
440
441 def __iter__(self):
442 return iter(self._packages_packages)
443
444 def __eq__(self, other):
445 if not isinstance(other, type(self)):
446 return False
447
448 return self._packages_packages == other._packages
449
450 def update(self, other):
451 """Update packages with contents of another set of packages.
452
453 Parameters
454 ----------
455 other : `Packages`
456 Other packages to merge with self.
457
458 Notes
459 -----
460 No check is made to see if we're clobbering anything.
461 """
462 self._packages_packages.update(other._packages)
463 self._names_names.update(other._names)
464
465 def extra(self, other):
466 """Get packages in self but not in another `Packages` object.
467
468 Parameters
469 ----------
470 other : `Packages`
471 Other packages to compare against.
472
473 Returns
474 -------
475 extra : `dict`
476 Extra packages. Keys (type `str`) are package names; values
477 (type `str`) are their versions.
478 """
479 return {pkg: self._packages_packages[pkg] for pkg in self._names_names - other._names}
480
481 def missing(self, other):
482 """Get packages in another `Packages` object but missing from self.
483
484 Parameters
485 ----------
486 other : `Packages`
487 Other packages to compare against.
488
489 Returns
490 -------
491 missing : `dict`
492 Missing packages. Keys (type `str`) are package names; values
493 (type `str`) are their versions.
494 """
495 return {pkg: other._packages[pkg] for pkg in other._names - self._names_names}
496
497 def difference(self, other):
498 """Get packages in symmetric difference of self and another `Packages`
499 object.
500
501 Parameters
502 ----------
503 other : `Packages`
504 Other packages to compare against.
505
506 Returns
507 -------
508 difference : `dict`
509 Packages in symmetric difference. Keys (type `str`) are package
510 names; values (type `str`) are their versions.
511 """
512 return {pkg: (self._packages_packages[pkg], other._packages[pkg]) for
513 pkg in self._names_names & other._names if self._packages_packages[pkg] != other._packages[pkg]}
514
515
516# Register YAML representers
517
518def pkg_representer(dumper, data):
519 """Represent Packages as a simple dict"""
520 return dumper.represent_mapping("lsst.base.Packages", data._packages,
521 flow_style=None)
522
523
524yaml.add_representer(Packages, pkg_representer)
525
526
527def pkg_constructor(loader, node):
528 yield Packages(loader.construct_mapping(node, deep=True))
529
530
531for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
532 yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)
def fromBytes(cls, data, format)
Definition: packages.py:341
def __contains__(self, pkg)
Definition: packages.py:438
def read(cls, filename)
Definition: packages.py:362
def missing(self, other)
Definition: packages.py:481
def write(self, filename)
Definition: packages.py:406
def difference(self, other)
Definition: packages.py:497
def extra(self, other)
Definition: packages.py:465
def __init__(self, packages)
Definition: packages.py:317
def update(self, other)
Definition: packages.py:450
def toBytes(self, format)
Definition: packages.py:385
def __eq__(self, other)
Definition: packages.py:444
def getEnvironmentPackages()
Definition: packages.py:154
def getCondaPackages()
Definition: packages.py:223
def getPythonPackages()
Definition: packages.py:92
def pkg_representer(dumper, data)
Definition: packages.py:518
def getVersionFromPythonModule(module)
Definition: packages.py:57
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54