Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21""" 

22Determine which packages are being used in the system and their versions 

23""" 

24import os 

25import sys 

26import hashlib 

27import importlib 

28import subprocess 

29import logging 

30import pickle as pickle 

31import re 

32import yaml 

33from collections.abc import Mapping 

34from functools import lru_cache 

35 

36from .versions import getRuntimeVersions 

37 

38log = logging.getLogger(__name__) 

39 

40__all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", 

41 "getCondaPackages", "Packages"] 

42 

43 

44# Packages used at build-time (e.g., header-only) 

45BUILDTIME = set(["boost", "eigen", "tmv"]) 

46 

47# Python modules to attempt to load so we can try to get the version 

48# We do this because the version only appears to be available from python, 

49# but we use the library 

50PYTHON = set(["galsim"]) 

51 

52# Packages that don't seem to have a mechanism for reporting the runtime 

53# version. We need to guess the version from the environment 

54ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"]) 

55 

56 

57def getVersionFromPythonModule(module): 

58 """Determine the version of a python module. 

59 

60 Parameters 

61 ---------- 

62 module : `module` 

63 Module for which to get version. 

64 

65 Returns 

66 ------- 

67 version : `str` 

68 

69 Raises 

70 ------ 

71 AttributeError 

72 Raised if __version__ attribute is not set. 

73 

74 Notes 

75 ----- 

76 We supplement the version with information from the 

77 ``__dependency_versions__`` (a specific variable set by LSST's 

78 `~lsst.sconsUtils` at build time) only for packages that are typically 

79 used only at build-time. 

80 """ 

81 version = module.__version__ 

82 if hasattr(module, "__dependency_versions__"): 

83 # Add build-time dependencies 

84 deps = module.__dependency_versions__ 

85 buildtime = BUILDTIME & set(deps.keys()) 

86 if buildtime: 

87 version += " with " + " ".join("%s=%s" % (pkg, deps[pkg]) 

88 for pkg in sorted(buildtime)) 

89 return str(version) 

90 

91 

92def getPythonPackages(): 

93 """Get imported python packages and their versions. 

94 

95 Returns 

96 ------- 

97 packages : `dict` 

98 Keys (type `str`) are package names; values (type `str`) are their 

99 versions. 

100 

101 Notes 

102 ----- 

103 We wade through `sys.modules` and attempt to determine the version for each 

104 module. Note, therefore, that we can only report on modules that have 

105 *already* been imported. 

106 

107 We don't include any module for which we cannot determine a version. 

108 """ 

109 # Attempt to import libraries that only report their version in python 

110 for module in PYTHON: 

111 try: 

112 importlib.import_module(module) 

113 except Exception: 

114 pass # It's not available, so don't care 

115 

116 packages = {"python": sys.version} 

117 # Not iterating with sys.modules.iteritems() because it's not atomic and 

118 # subject to race conditions 

119 moduleNames = list(sys.modules.keys()) 

120 for name in moduleNames: 

121 module = sys.modules[name] 

122 try: 

123 ver = getVersionFromPythonModule(module) 

124 except Exception: 

125 continue # Can't get a version from it, don't care 

126 

127 # Remove "foo.bar.version" in favor of "foo.bar" 

128 # This prevents duplication when the __init__.py includes 

129 # "from .version import *" 

130 for ending in (".version", "._version"): 

131 if name.endswith(ending): 

132 name = name[:-len(ending)] 

133 if name in packages: 

134 assert ver == packages[name] 

135 elif name in packages: 

136 assert ver == packages[name] 

137 

138 # Use LSST package names instead of python module names 

139 # This matches the names we get from the environment (i.e., EUPS) 

140 # so we can clobber these build-time versions if the environment 

141 # reveals that we're not using the packages as-built. 

142 if "lsst" in name: 

143 name = name.replace("lsst.", "").replace(".", "_") 

144 

145 packages[name] = ver 

146 

147 return packages 

148 

149 

150_eups = None # Singleton Eups object 

151 

152 

153@lru_cache(maxsize=1) 

154def getEnvironmentPackages(): 

155 """Get products and their versions from the environment. 

156 

157 Returns 

158 ------- 

159 packages : `dict` 

160 Keys (type `str`) are product names; values (type `str`) are their 

161 versions. 

162 

163 Notes 

164 ----- 

165 We use EUPS to determine the version of certain products (those that don't 

166 provide a means to determine the version any other way) and to check if 

167 uninstalled packages are being used. We only report the product/version 

168 for these packages. 

169 """ 

170 try: 

171 from eups import Eups 

172 from eups.Product import Product 

173 except ImportError: 

174 log.warning("Unable to import eups, so cannot determine package versions from environment") 

175 return {} 

176 

177 # Cache eups object since creating it can take a while 

178 global _eups 

179 if not _eups: 

180 _eups = Eups() 

181 products = _eups.findProducts(tags=["setup"]) 

182 

183 # Get versions for things we can't determine via runtime mechanisms 

184 # XXX Should we just grab everything we can, rather than just a 

185 # predetermined set? 

186 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT} 

187 

188 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the 

189 # version name indicates uninstalled code, so the version could be 

190 # different than what's being reported by the runtime environment (because 

191 # we don't tend to run "scons" every time we update some python file, 

192 # and even if we did sconsUtils probably doesn't check to see if the repo 

193 # is clean). 

194 for prod in products: 

195 if not prod.version.startswith(Product.LocalVersionPrefix): 

196 continue 

197 ver = prod.version 

198 

199 gitDir = os.path.join(prod.dir, ".git") 

200 if os.path.exists(gitDir): 

201 # get the git revision and an indication if the working copy is 

202 # clean 

203 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"] 

204 diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff", 

205 "--patch"] 

206 try: 

207 rev = subprocess.check_output(revCmd).decode().strip() 

208 diff = subprocess.check_output(diffCmd) 

209 except Exception: 

210 ver += "@GIT_ERROR" 

211 else: 

212 ver += "@" + rev 

213 if diff: 

214 ver += "+" + hashlib.md5(diff).hexdigest() 

215 else: 

216 ver += "@NO_GIT" 

217 

218 packages[prod.name] = ver 

219 return packages 

220 

221 

222@lru_cache(maxsize=1) 

223def getCondaPackages(): 

224 """Get products and their versions from the conda environment. 

225 

226 Returns 

227 ------- 

228 packages : `dict` 

229 Keys (type `str`) are product names; values (type `str`) are their 

230 versions. 

231 

232 Notes 

233 ----- 

234 Returns empty result if a conda environment is not in use or can not 

235 be queried. 

236 """ 

237 

238 try: 

239 import json 

240 from conda.cli.python_api import Commands, run_command 

241 except ImportError: 

242 return {} 

243 

244 # Get the installed package list 

245 versions_json = run_command(Commands.LIST, "--json") 

246 packages = {pkg["name"]: pkg["version"] for pkg in json.loads(versions_json[0])} 

247 

248 # Try to work out the conda environment name and include it as a fake 

249 # package. The "obvious" way of running "conda info --json" does give 

250 # access to the active_prefix but takes about 2 seconds to run. 

251 # The equivalent to the code above would be: 

252 # info_json = run_command(Commands.INFO, "--json") 

253 # As a comporomise look for the env name in the path to the python 

254 # executable 

255 match = re.search(r"/envs/(.*?)/bin/", sys.executable) 

256 if match: 

257 packages["conda_env"] = match.group(1) 

258 

259 return packages 

260 

261 

262class Packages: 

263 """A table of packages and their versions. 

264 

265 There are a few different types of packages, and their versions are 

266 collected in different ways: 

267 

268 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from 

269 interrogating the dynamic library 

270 2. Python modules (e.g., afw, numpy; galsim is also in this group even 

271 though we only use it through the library, because no version 

272 information is currently provided through the library): we get their 

273 version from the ``__version__`` module variable. Note that this means 

274 that we're only aware of modules that have already been imported. 

275 3. Other packages provide no run-time accessible version information (e.g., 

276 astrometry_net): we get their version from interrogating the 

277 environment. Currently, that means EUPS; if EUPS is replaced or dropped 

278 then we'll need to consider an alternative means of getting this version 

279 information. 

280 4. Local versions of packages (a non-installed EUPS package, selected with 

281 ``setup -r /path/to/package``): we identify these through the 

282 environment (EUPS again) and use as a version the path supplemented with 

283 the ``git`` SHA and, if the git repo isn't clean, an MD5 of the diff. 

284 

285 These package versions are collected and stored in a Packages object, which 

286 provides useful comparison and persistence features. 

287 

288 Example usage: 

289 

290 .. code-block:: python 

291 

292 from lsst.base import Packages 

293 pkgs = Packages.fromSystem() 

294 print("Current packages:", pkgs) 

295 old = Packages.read("/path/to/packages.pickle") 

296 print("Old packages:", old) 

297 print("Missing packages compared to before:", pkgs.missing(old)) 

298 print("Extra packages compared to before:", pkgs.extra(old)) 

299 print("Different packages: ", pkgs.difference(old)) 

300 old.update(pkgs) # Include any new packages in the old 

301 old.write("/path/to/packages.pickle") 

302 

303 Parameters 

304 ---------- 

305 packages : `dict` 

306 A mapping {package: version} where both keys and values are type `str`. 

307 

308 Notes 

309 ----- 

310 This is essentially a wrapper around a dict with some conveniences. 

311 """ 

312 

313 formats = {".pkl": "pickle", 

314 ".pickle": "pickle", 

315 ".yaml": "yaml"} 

316 

317 def __init__(self, packages): 

318 assert isinstance(packages, Mapping) 

319 self._packages = packages 

320 self._names = set(packages.keys()) 

321 

322 @classmethod 

323 def fromSystem(cls): 

324 """Construct a `Packages` by examining the system. 

325 

326 Determine packages by examining python's `sys.modules`, runtime 

327 libraries and EUPS. 

328 

329 Returns 

330 ------- 

331 packages : `Packages` 

332 """ 

333 packages = {} 

334 packages.update(getPythonPackages()) 

335 packages.update(getCondaPackages()) 

336 packages.update(getRuntimeVersions()) 

337 packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions 

338 return cls(packages) 

339 

340 @classmethod 

341 def fromBytes(cls, data, format): 

342 """Construct the object from a byte representation. 

343 

344 Parameters 

345 ---------- 

346 data : `bytes` 

347 The serialized form of this object in bytes. 

348 format : `str` 

349 The format of those bytes. Can be ``yaml`` or ``pickle``. 

350 """ 

351 if format == "pickle": 

352 new = pickle.loads(data) 

353 elif format == "yaml": 

354 new = yaml.load(data, Loader=yaml.SafeLoader) 

355 else: 

356 raise ValueError(f"Unexpected serialization format given: {format}") 

357 if not isinstance(new, cls): 

358 raise TypeError(f"Extracted object of class '{type(new)}' but expected '{cls}'") 

359 return new 

360 

361 @classmethod 

362 def read(cls, filename): 

363 """Read packages from filename. 

364 

365 Parameters 

366 ---------- 

367 filename : `str` 

368 Filename from which to read. The format is determined from the 

369 file extension. Currently support ``.pickle``, ``.pkl`` 

370 and ``.yaml``. 

371 

372 Returns 

373 ------- 

374 packages : `Packages` 

375 """ 

376 _, ext = os.path.splitext(filename) 

377 if ext not in cls.formats: 

378 raise ValueError(f"Format from {ext} extension in file {filename} not recognized") 

379 with open(filename, "rb") as ff: 

380 # We assume that these classes are tiny so there is no 

381 # substantive memory impact by reading the entire file up front 

382 data = ff.read() 

383 return cls.fromBytes(data, cls.formats[ext]) 

384 

385 def toBytes(self, format): 

386 """Convert the object to a serialized bytes form using the 

387 specified format. 

388 

389 Parameters 

390 ---------- 

391 format : `str` 

392 Format to use when serializing. Can be ``yaml`` or ``pickle``. 

393 

394 Returns 

395 ------- 

396 data : `bytes` 

397 Byte string representing the serialized object. 

398 """ 

399 if format == "pickle": 

400 return pickle.dumps(self) 

401 elif format == "yaml": 

402 return yaml.dump(self).encode("utf-8") 

403 else: 

404 raise ValueError(f"Unexpected serialization format requested: {format}") 

405 

406 def write(self, filename): 

407 """Write to file. 

408 

409 Parameters 

410 ---------- 

411 filename : `str` 

412 Filename to which to write. The format of the data file 

413 is determined from the file extension. Currently supports 

414 ``.pickle`` and ``.yaml`` 

415 """ 

416 _, ext = os.path.splitext(filename) 

417 if ext not in self.formats: 

418 raise ValueError(f"Format from {ext} extension in file {filename} not recognized") 

419 with open(filename, "wb") as ff: 

420 # Assumes that the bytes serialization of this object is 

421 # relatively small. 

422 ff.write(self.toBytes(self.formats[ext])) 

423 

424 def __len__(self): 

425 return len(self._packages) 

426 

427 def __str__(self): 

428 ss = "%s({\n" % self.__class__.__name__ 

429 # Sort alphabetically by module name, for convenience in reading 

430 ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for 

431 prod in sorted(self._names)) 

432 ss += ",\n})" 

433 return ss 

434 

435 def __repr__(self): 

436 return "%s(%s)" % (self.__class__.__name__, repr(self._packages)) 

437 

438 def __contains__(self, pkg): 

439 return pkg in self._packages 

440 

441 def __iter__(self): 

442 return iter(self._packages) 

443 

444 def __eq__(self, other): 

445 if not isinstance(other, type(self)): 

446 return False 

447 

448 return self._packages == other._packages 

449 

450 def update(self, other): 

451 """Update packages with contents of another set of packages. 

452 

453 Parameters 

454 ---------- 

455 other : `Packages` 

456 Other packages to merge with self. 

457 

458 Notes 

459 ----- 

460 No check is made to see if we're clobbering anything. 

461 """ 

462 self._packages.update(other._packages) 

463 self._names.update(other._names) 

464 

465 def extra(self, other): 

466 """Get packages in self but not in another `Packages` object. 

467 

468 Parameters 

469 ---------- 

470 other : `Packages` 

471 Other packages to compare against. 

472 

473 Returns 

474 ------- 

475 extra : `dict` 

476 Extra packages. Keys (type `str`) are package names; values 

477 (type `str`) are their versions. 

478 """ 

479 return {pkg: self._packages[pkg] for pkg in self._names - other._names} 

480 

481 def missing(self, other): 

482 """Get packages in another `Packages` object but missing from self. 

483 

484 Parameters 

485 ---------- 

486 other : `Packages` 

487 Other packages to compare against. 

488 

489 Returns 

490 ------- 

491 missing : `dict` 

492 Missing packages. Keys (type `str`) are package names; values 

493 (type `str`) are their versions. 

494 """ 

495 return {pkg: other._packages[pkg] for pkg in other._names - self._names} 

496 

497 def difference(self, other): 

498 """Get packages in symmetric difference of self and another `Packages` 

499 object. 

500 

501 Parameters 

502 ---------- 

503 other : `Packages` 

504 Other packages to compare against. 

505 

506 Returns 

507 ------- 

508 difference : `dict` 

509 Packages in symmetric difference. Keys (type `str`) are package 

510 names; values (type `str`) are their versions. 

511 """ 

512 return {pkg: (self._packages[pkg], other._packages[pkg]) for 

513 pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]} 

514 

515 

516# Register YAML representers 

517 

518def pkg_representer(dumper, data): 

519 """Represent Packages as a simple dict""" 

520 return dumper.represent_mapping("lsst.base.Packages", data._packages, 

521 flow_style=None) 

522 

523 

524yaml.add_representer(Packages, pkg_representer) 

525 

526 

527def pkg_constructor(loader, node): 

528 yield Packages(loader.construct_mapping(node, deep=True)) 

529 

530 

531for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader): 

532 yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)