Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21""" 

22Determine which packages are being used in the system and their versions 

23""" 

24import os 

25import sys 

26import hashlib 

27import importlib 

28import subprocess 

29import logging 

30import pickle as pickle 

31import yaml 

32from collections.abc import Mapping 

33 

34from .versions import getRuntimeVersions 

35 

36log = logging.getLogger(__name__) 

37 

38__all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"] 

39 

40 

41# Packages used at build-time (e.g., header-only) 

42BUILDTIME = set(["boost", "eigen", "tmv"]) 

43 

44# Python modules to attempt to load so we can try to get the version 

45# We do this because the version only appears to be available from python, but we use the library 

46PYTHON = set(["galsim"]) 

47 

48# Packages that don't seem to have a mechanism for reporting the runtime version 

49# We need to guess the version from the environment 

50ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"]) 

51 

52 

53def getVersionFromPythonModule(module): 

54 """Determine the version of a python module. 

55 

56 Parameters 

57 ---------- 

58 module : `module` 

59 Module for which to get version. 

60 

61 Returns 

62 ------- 

63 version : `str` 

64 

65 Raises 

66 ------ 

67 AttributeError 

68 Raised if __version__ attribute is not set. 

69 

70 Notes 

71 ----- 

72 We supplement the version with information from the 

73 ``__dependency_versions__`` (a specific variable set by LSST's 

74 `~lsst.sconsUtils` at build time) only for packages that are typically 

75 used only at build-time. 

76 """ 

77 version = module.__version__ 

78 if hasattr(module, "__dependency_versions__"): 

79 # Add build-time dependencies 

80 deps = module.__dependency_versions__ 

81 buildtime = BUILDTIME & set(deps.keys()) 

82 if buildtime: 

83 version += " with " + " ".join("%s=%s" % (pkg, deps[pkg]) 

84 for pkg in sorted(buildtime)) 

85 return version 

86 

87 

88def getPythonPackages(): 

89 """Get imported python packages and their versions. 

90 

91 Returns 

92 ------- 

93 packages : `dict` 

94 Keys (type `str`) are package names; values (type `str`) are their 

95 versions. 

96 

97 Notes 

98 ----- 

99 We wade through `sys.modules` and attempt to determine the version for each 

100 module. Note, therefore, that we can only report on modules that have 

101 *already* been imported. 

102 

103 We don't include any module for which we cannot determine a version. 

104 """ 

105 # Attempt to import libraries that only report their version in python 

106 for module in PYTHON: 

107 try: 

108 importlib.import_module(module) 

109 except Exception: 

110 pass # It's not available, so don't care 

111 

112 packages = {"python": sys.version} 

113 # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions 

114 moduleNames = list(sys.modules.keys()) 

115 for name in moduleNames: 

116 module = sys.modules[name] 

117 try: 

118 ver = getVersionFromPythonModule(module) 

119 except Exception: 

120 continue # Can't get a version from it, don't care 

121 

122 # Remove "foo.bar.version" in favor of "foo.bar" 

123 # This prevents duplication when the __init__.py includes "from .version import *" 

124 for ending in (".version", "._version"): 

125 if name.endswith(ending): 

126 name = name[:-len(ending)] 

127 if name in packages: 

128 assert ver == packages[name] 

129 elif name in packages: 

130 assert ver == packages[name] 

131 

132 # Use LSST package names instead of python module names 

133 # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time 

134 # versions if the environment reveals that we're not using the packages as-built. 

135 if "lsst" in name: 

136 name = name.replace("lsst.", "").replace(".", "_") 

137 

138 packages[name] = ver 

139 

140 return packages 

141 

142 

143_eups = None # Singleton Eups object 

144 

145 

146def getEnvironmentPackages(): 

147 """Get products and their versions from the environment. 

148 

149 Returns 

150 ------- 

151 packages : `dict` 

152 Keys (type `str`) are product names; values (type `str`) are their 

153 versions. 

154 

155 Notes 

156 ----- 

157 We use EUPS to determine the version of certain products (those that don't 

158 provide a means to determine the version any other way) and to check if 

159 uninstalled packages are being used. We only report the product/version 

160 for these packages. 

161 """ 

162 try: 

163 from eups import Eups 

164 from eups.Product import Product 

165 except ImportError: 

166 log.warning("Unable to import eups, so cannot determine package versions from environment") 

167 return {} 

168 

169 # Cache eups object since creating it can take a while 

170 global _eups 

171 if not _eups: 

172 _eups = Eups() 

173 products = _eups.findProducts(tags=["setup"]) 

174 

175 # Get versions for things we can't determine via runtime mechanisms 

176 # XXX Should we just grab everything we can, rather than just a predetermined set? 

177 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT} 

178 

179 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled 

180 # code, so the version could be different than what's being reported by the runtime environment (because 

181 # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils 

182 # probably doesn't check to see if the repo is clean). 

183 for prod in products: 

184 if not prod.version.startswith(Product.LocalVersionPrefix): 

185 continue 

186 ver = prod.version 

187 

188 gitDir = os.path.join(prod.dir, ".git") 

189 if os.path.exists(gitDir): 

190 # get the git revision and an indication if the working copy is clean 

191 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"] 

192 diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff", 

193 "--patch"] 

194 try: 

195 rev = subprocess.check_output(revCmd).decode().strip() 

196 diff = subprocess.check_output(diffCmd) 

197 except Exception: 

198 ver += "@GIT_ERROR" 

199 else: 

200 ver += "@" + rev 

201 if diff: 

202 ver += "+" + hashlib.md5(diff).hexdigest() 

203 else: 

204 ver += "@NO_GIT" 

205 

206 packages[prod.name] = ver 

207 return packages 

208 

209 

210class Packages: 

211 """A table of packages and their versions. 

212 

213 There are a few different types of packages, and their versions are collected 

214 in different ways: 

215 

216 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from 

217 interrogating the dynamic library 

218 2. Python modules (e.g., afw, numpy; galsim is also in this group even though 

219 we only use it through the library, because no version information is 

220 currently provided through the library): we get their version from the 

221 ``__version__`` module variable. Note that this means that we're only aware 

222 of modules that have already been imported. 

223 3. Other packages provide no run-time accessible version information (e.g., 

224 astrometry_net): we get their version from interrogating the environment. 

225 Currently, that means EUPS; if EUPS is replaced or dropped then we'll need 

226 to consider an alternative means of getting this version information. 

227 4. Local versions of packages (a non-installed EUPS package, selected with 

228 ``setup -r /path/to/package``): we identify these through the environment 

229 (EUPS again) and use as a version the path supplemented with the ``git`` 

230 SHA and, if the git repo isn't clean, an MD5 of the diff. 

231 

232 These package versions are collected and stored in a Packages object, which 

233 provides useful comparison and persistence features. 

234 

235 Example usage: 

236 

237 .. code-block:: python 

238 

239 from lsst.base import Packages 

240 pkgs = Packages.fromSystem() 

241 print("Current packages:", pkgs) 

242 old = Packages.read("/path/to/packages.pickle") 

243 print("Old packages:", old) 

244 print("Missing packages compared to before:", pkgs.missing(old)) 

245 print("Extra packages compared to before:", pkgs.extra(old)) 

246 print("Different packages: ", pkgs.difference(old)) 

247 old.update(pkgs) # Include any new packages in the old 

248 old.write("/path/to/packages.pickle") 

249 

250 Parameters 

251 ---------- 

252 packages : `dict` 

253 A mapping {package: version} where both keys and values are type `str`. 

254 

255 Notes 

256 ----- 

257 This is essentially a wrapper around a dict with some conveniences. 

258 """ 

259 

260 formats = {".pkl": "pickle", 

261 ".pickle": "pickle", 

262 ".yaml": "yaml"} 

263 

264 def __init__(self, packages): 

265 assert isinstance(packages, Mapping) 

266 self._packages = packages 

267 self._names = set(packages.keys()) 

268 

269 @classmethod 

270 def fromSystem(cls): 

271 """Construct a `Packages` by examining the system. 

272 

273 Determine packages by examining python's `sys.modules`, runtime 

274 libraries and EUPS. 

275 

276 Returns 

277 ------- 

278 packages : `Packages` 

279 """ 

280 packages = {} 

281 packages.update(getPythonPackages()) 

282 packages.update(getRuntimeVersions()) 

283 packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions 

284 return cls(packages) 

285 

286 @classmethod 

287 def fromBytes(cls, data, format): 

288 """Construct the object from a byte representation. 

289 

290 Parameters 

291 ---------- 

292 data : `bytes` 

293 The serialized form of this object in bytes. 

294 format : `str` 

295 The format of those bytes. Can be ``yaml`` or ``pickle``. 

296 """ 

297 if format == "pickle": 

298 new = pickle.loads(data) 

299 elif format == "yaml": 

300 new = yaml.load(data, Loader=yaml.SafeLoader) 

301 else: 

302 raise ValueError(f"Unexpected serialization format given: {format}") 

303 if not isinstance(new, cls): 

304 raise TypeError(f"Extracted object of class '{type(new)}' but expected '{cls}'") 

305 return new 

306 

307 @classmethod 

308 def read(cls, filename): 

309 """Read packages from filename. 

310 

311 Parameters 

312 ---------- 

313 filename : `str` 

314 Filename from which to read. The format is determined from the 

315 file extension. Currently support ``.pickle``, ``.pkl`` 

316 and ``.yaml``. 

317 

318 Returns 

319 ------- 

320 packages : `Packages` 

321 """ 

322 _, ext = os.path.splitext(filename) 

323 if ext not in cls.formats: 

324 raise ValueError(f"Format from {ext} extension in file {filename} not recognized") 

325 with open(filename, "rb") as ff: 

326 # We assume that these classes are tiny so there is no 

327 # substantive memory impact by reading the entire file up front 

328 data = ff.read() 

329 return cls.fromBytes(data, cls.formats[ext]) 

330 

331 def toBytes(self, format): 

332 """Convert the object to a serialized bytes form using the 

333 specified format. 

334 

335 Parameters 

336 ---------- 

337 format : `str` 

338 Format to use when serializing. Can be ``yaml`` or ``pickle``. 

339 

340 Returns 

341 ------- 

342 data : `bytes` 

343 Byte string representing the serialized object. 

344 """ 

345 if format == "pickle": 

346 return pickle.dumps(self) 

347 elif format == "yaml": 

348 return yaml.dump(self).encode("utf-8") 

349 else: 

350 raise ValueError(f"Unexpected serialization format requested: {format}") 

351 

352 def write(self, filename): 

353 """Write to file. 

354 

355 Parameters 

356 ---------- 

357 filename : `str` 

358 Filename to which to write. The format of the data file 

359 is determined from the file extension. Currently supports 

360 ``.pickle`` and ``.yaml`` 

361 """ 

362 _, ext = os.path.splitext(filename) 

363 if ext not in self.formats: 

364 raise ValueError(f"Format from {ext} extension in file {filename} not recognized") 

365 with open(filename, "wb") as ff: 

366 # Assumes that the bytes serialization of this object is 

367 # relatively small. 

368 ff.write(self.toBytes(self.formats[ext])) 

369 

370 def __len__(self): 

371 return len(self._packages) 

372 

373 def __str__(self): 

374 ss = "%s({\n" % self.__class__.__name__ 

375 # Sort alphabetically by module name, for convenience in reading 

376 ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for 

377 prod in sorted(self._names)) 

378 ss += ",\n})" 

379 return ss 

380 

381 def __repr__(self): 

382 return "%s(%s)" % (self.__class__.__name__, repr(self._packages)) 

383 

384 def __contains__(self, pkg): 

385 return pkg in self._packages 

386 

387 def __iter__(self): 

388 return iter(self._packages) 

389 

390 def __eq__(self, other): 

391 if not isinstance(other, type(self)): 

392 return False 

393 

394 return self._packages == other._packages 

395 

396 def update(self, other): 

397 """Update packages with contents of another set of packages. 

398 

399 Parameters 

400 ---------- 

401 other : `Packages` 

402 Other packages to merge with self. 

403 

404 Notes 

405 ----- 

406 No check is made to see if we're clobbering anything. 

407 """ 

408 self._packages.update(other._packages) 

409 self._names.update(other._names) 

410 

411 def extra(self, other): 

412 """Get packages in self but not in another `Packages` object. 

413 

414 Parameters 

415 ---------- 

416 other : `Packages` 

417 Other packages to compare against. 

418 

419 Returns 

420 ------- 

421 extra : `dict` 

422 Extra packages. Keys (type `str`) are package names; values 

423 (type `str`) are their versions. 

424 """ 

425 return {pkg: self._packages[pkg] for pkg in self._names - other._names} 

426 

427 def missing(self, other): 

428 """Get packages in another `Packages` object but missing from self. 

429 

430 Parameters 

431 ---------- 

432 other : `Packages` 

433 Other packages to compare against. 

434 

435 Returns 

436 ------- 

437 missing : `dict` 

438 Missing packages. Keys (type `str`) are package names; values 

439 (type `str`) are their versions. 

440 """ 

441 return {pkg: other._packages[pkg] for pkg in other._names - self._names} 

442 

443 def difference(self, other): 

444 """Get packages in symmetric difference of self and another `Packages` 

445 object. 

446 

447 Parameters 

448 ---------- 

449 other : `Packages` 

450 Other packages to compare against. 

451 

452 Returns 

453 ------- 

454 difference : `dict` 

455 Packages in symmetric difference. Keys (type `str`) are package 

456 names; values (type `str`) are their versions. 

457 """ 

458 return {pkg: (self._packages[pkg], other._packages[pkg]) for 

459 pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]} 

460 

461 

462# Register YAML representers 

463 

464def pkg_representer(dumper, data): 

465 """Represent Packages as a simple dict""" 

466 return dumper.represent_mapping("lsst.base.Packages", data._packages, 

467 flow_style=None) 

468 

469 

470yaml.add_representer(Packages, pkg_representer) 

471 

472 

473def pkg_constructor(loader, node): 

474 yield Packages(loader.construct_mapping(node, deep=True)) 

475 

476 

477for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader): 

478 yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)