Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21""" 

22Determine which packages are being used in the system and their versions 

23""" 

24import os 

25import sys 

26import hashlib 

27import importlib 

28import subprocess 

29import logging 

30import pickle as pickle 

31import yaml 

32from collections.abc import Mapping 

33 

34from .versions import getRuntimeVersions 

35 

36log = logging.getLogger(__name__) 

37 

38__all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"] 

39 

40 

41# Packages used at build-time (e.g., header-only) 

42BUILDTIME = set(["boost", "eigen", "tmv"]) 

43 

44# Python modules to attempt to load so we can try to get the version 

45# We do this because the version only appears to be available from python, but we use the library 

46PYTHON = set(["galsim"]) 

47 

48# Packages that don't seem to have a mechanism for reporting the runtime version 

49# We need to guess the version from the environment 

50ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"]) 

51 

52 

53def getVersionFromPythonModule(module): 

54 """Determine the version of a python module. 

55 

56 Parameters 

57 ---------- 

58 module : `module` 

59 Module for which to get version. 

60 

61 Returns 

62 ------- 

63 version : `str` 

64 

65 Raises 

66 ------ 

67 AttributeError 

68 Raised if __version__ attribute is not set. 

69 

70 Notes 

71 ----- 

72 We supplement the version with information from the 

73 ``__dependency_versions__`` (a specific variable set by LSST's 

74 `~lsst.sconsUtils` at build time) only for packages that are typically 

75 used only at build-time. 

76 """ 

77 version = module.__version__ 

78 if hasattr(module, "__dependency_versions__"): 

79 # Add build-time dependencies 

80 deps = module.__dependency_versions__ 

81 buildtime = BUILDTIME & set(deps.keys()) 

82 if buildtime: 

83 version += " with " + " ".join("%s=%s" % (pkg, deps[pkg]) 

84 for pkg in sorted(buildtime)) 

85 return version 

86 

87 

88def getPythonPackages(): 

89 """Get imported python packages and their versions. 

90 

91 Returns 

92 ------- 

93 packages : `dict` 

94 Keys (type `str`) are package names; values (type `str`) are their 

95 versions. 

96 

97 Notes 

98 ----- 

99 We wade through `sys.modules` and attempt to determine the version for each 

100 module. Note, therefore, that we can only report on modules that have 

101 *already* been imported. 

102 

103 We don't include any module for which we cannot determine a version. 

104 """ 

105 # Attempt to import libraries that only report their version in python 

106 for module in PYTHON: 

107 try: 

108 importlib.import_module(module) 

109 except Exception: 

110 pass # It's not available, so don't care 

111 

112 packages = {"python": sys.version} 

113 # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions 

114 moduleNames = list(sys.modules.keys()) 

115 for name in moduleNames: 

116 module = sys.modules[name] 

117 try: 

118 ver = getVersionFromPythonModule(module) 

119 except Exception: 

120 continue # Can't get a version from it, don't care 

121 

122 # Remove "foo.bar.version" in favor of "foo.bar" 

123 # This prevents duplication when the __init__.py includes "from .version import *" 

124 for ending in (".version", "._version"): 

125 if name.endswith(ending): 

126 name = name[:-len(ending)] 

127 if name in packages: 

128 assert ver == packages[name] 

129 elif name in packages: 

130 assert ver == packages[name] 

131 

132 # Use LSST package names instead of python module names 

133 # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time 

134 # versions if the environment reveals that we're not using the packages as-built. 

135 if "lsst" in name: 

136 name = name.replace("lsst.", "").replace(".", "_") 

137 

138 packages[name] = ver 

139 

140 return packages 

141 

142 

143_eups = None # Singleton Eups object 

144 

145 

146def getEnvironmentPackages(): 

147 """Get products and their versions from the environment. 

148 

149 Returns 

150 ------- 

151 packages : `dict` 

152 Keys (type `str`) are product names; values (type `str`) are their 

153 versions. 

154 

155 Notes 

156 ----- 

157 We use EUPS to determine the version of certain products (those that don't 

158 provide a means to determine the version any other way) and to check if 

159 uninstalled packages are being used. We only report the product/version 

160 for these packages. 

161 """ 

162 try: 

163 from eups import Eups 

164 from eups.Product import Product 

165 except ImportError: 

166 log.warning("Unable to import eups, so cannot determine package versions from environment") 

167 return {} 

168 

169 # Cache eups object since creating it can take a while 

170 global _eups 

171 if not _eups: 

172 _eups = Eups() 

173 products = _eups.findProducts(tags=["setup"]) 

174 

175 # Get versions for things we can't determine via runtime mechanisms 

176 # XXX Should we just grab everything we can, rather than just a predetermined set? 

177 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT} 

178 

179 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled 

180 # code, so the version could be different than what's being reported by the runtime environment (because 

181 # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils 

182 # probably doesn't check to see if the repo is clean). 

183 for prod in products: 

184 if not prod.version.startswith(Product.LocalVersionPrefix): 

185 continue 

186 ver = prod.version 

187 

188 gitDir = os.path.join(prod.dir, ".git") 

189 if os.path.exists(gitDir): 

190 # get the git revision and an indication if the working copy is clean 

191 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"] 

192 diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff", 

193 "--patch"] 

194 try: 

195 rev = subprocess.check_output(revCmd).decode().strip() 

196 diff = subprocess.check_output(diffCmd) 

197 except Exception: 

198 ver += "@GIT_ERROR" 

199 else: 

200 ver += "@" + rev 

201 if diff: 

202 ver += "+" + hashlib.md5(diff).hexdigest() 

203 else: 

204 ver += "@NO_GIT" 

205 

206 packages[prod.name] = ver 

207 return packages 

208 

209 

210class Packages: 

211 """A table of packages and their versions. 

212 

213 There are a few different types of packages, and their versions are collected 

214 in different ways: 

215 

216 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from 

217 interrogating the dynamic library 

218 2. Python modules (e.g., afw, numpy; galsim is also in this group even though 

219 we only use it through the library, because no version information is 

220 currently provided through the library): we get their version from the 

221 ``__version__`` module variable. Note that this means that we're only aware 

222 of modules that have already been imported. 

223 3. Other packages provide no run-time accessible version information (e.g., 

224 astrometry_net): we get their version from interrogating the environment. 

225 Currently, that means EUPS; if EUPS is replaced or dropped then we'll need 

226 to consider an alternative means of getting this version information. 

227 4. Local versions of packages (a non-installed EUPS package, selected with 

228 ``setup -r /path/to/package``): we identify these through the environment 

229 (EUPS again) and use as a version the path supplemented with the ``git`` 

230 SHA and, if the git repo isn't clean, an MD5 of the diff. 

231 

232 These package versions are collected and stored in a Packages object, which 

233 provides useful comparison and persistence features. 

234 

235 Example usage: 

236 

237 .. code-block:: python 

238 

239 from lsst.base import Packages 

240 pkgs = Packages.fromSystem() 

241 print("Current packages:", pkgs) 

242 old = Packages.read("/path/to/packages.pickle") 

243 print("Old packages:", old) 

244 print("Missing packages compared to before:", pkgs.missing(old)) 

245 print("Extra packages compared to before:", pkgs.extra(old)) 

246 print("Different packages: ", pkgs.difference(old)) 

247 old.update(pkgs) # Include any new packages in the old 

248 old.write("/path/to/packages.pickle") 

249 

250 Parameters 

251 ---------- 

252 packages : `dict` 

253 A mapping {package: version} where both keys and values are type `str`. 

254 

255 Notes 

256 ----- 

257 This is essentially a wrapper around a dict with some conveniences. 

258 """ 

259 

260 def __init__(self, packages): 

261 assert isinstance(packages, Mapping) 

262 self._packages = packages 

263 self._names = set(packages.keys()) 

264 

265 @classmethod 

266 def fromSystem(cls): 

267 """Construct a `Packages` by examining the system. 

268 

269 Determine packages by examining python's `sys.modules`, runtime 

270 libraries and EUPS. 

271 

272 Returns 

273 ------- 

274 packages : `Packages` 

275 """ 

276 packages = {} 

277 packages.update(getPythonPackages()) 

278 packages.update(getRuntimeVersions()) 

279 packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions 

280 return cls(packages) 

281 

282 @classmethod 

283 def read(cls, filename): 

284 """Read packages from filename. 

285 

286 Parameters 

287 ---------- 

288 filename : `str` 

289 Filename from which to read. The format is determined from the 

290 file extension. Currently support ``.pickle``, ``.pkl`` 

291 and ``.yaml``. 

292 

293 Returns 

294 ------- 

295 packages : `Packages` 

296 """ 

297 _, ext = os.path.splitext(filename) 

298 if ext in (".pickle", ".pkl"): 

299 with open(filename, "rb") as ff: 

300 return pickle.load(ff) 

301 elif ext == ".yaml": 

302 with open(filename, "r") as ff: 

303 return yaml.load(ff, Loader=yaml.SafeLoader) 

304 else: 

305 raise ValueError(f"Unable to determine how to read file {filename} from extension {ext}") 

306 

307 def write(self, filename): 

308 """Write to file. 

309 

310 Parameters 

311 ---------- 

312 filename : `str` 

313 Filename to which to write. The format of the data file 

314 is determined from the file extension. Currently supports 

315 ``.pickle`` and ``.yaml`` 

316 """ 

317 _, ext = os.path.splitext(filename) 

318 if ext in (".pickle", ".pkl"): 

319 with open(filename, "wb") as ff: 

320 pickle.dump(self, ff) 

321 elif ext == ".yaml": 

322 with open(filename, "w") as ff: 

323 yaml.dump(self, ff) 

324 else: 

325 raise ValueError(f"Unexpected file format requested: {ext}") 

326 

327 def __len__(self): 

328 return len(self._packages) 

329 

330 def __str__(self): 

331 ss = "%s({\n" % self.__class__.__name__ 

332 # Sort alphabetically by module name, for convenience in reading 

333 ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for 

334 prod in sorted(self._names)) 

335 ss += ",\n})" 

336 return ss 

337 

338 def __repr__(self): 

339 return "%s(%s)" % (self.__class__.__name__, repr(self._packages)) 

340 

341 def __contains__(self, pkg): 

342 return pkg in self._packages 

343 

344 def __iter__(self): 

345 return iter(self._packages) 

346 

347 def __eq__(self, other): 

348 if not isinstance(other, type(self)): 

349 return False 

350 

351 return self._packages == other._packages 

352 

353 def update(self, other): 

354 """Update packages with contents of another set of packages. 

355 

356 Parameters 

357 ---------- 

358 other : `Packages` 

359 Other packages to merge with self. 

360 

361 Notes 

362 ----- 

363 No check is made to see if we're clobbering anything. 

364 """ 

365 self._packages.update(other._packages) 

366 self._names.update(other._names) 

367 

368 def extra(self, other): 

369 """Get packages in self but not in another `Packages` object. 

370 

371 Parameters 

372 ---------- 

373 other : `Packages` 

374 Other packages to compare against. 

375 

376 Returns 

377 ------- 

378 extra : `dict` 

379 Extra packages. Keys (type `str`) are package names; values 

380 (type `str`) are their versions. 

381 """ 

382 return {pkg: self._packages[pkg] for pkg in self._names - other._names} 

383 

384 def missing(self, other): 

385 """Get packages in another `Packages` object but missing from self. 

386 

387 Parameters 

388 ---------- 

389 other : `Packages` 

390 Other packages to compare against. 

391 

392 Returns 

393 ------- 

394 missing : `dict` 

395 Missing packages. Keys (type `str`) are package names; values 

396 (type `str`) are their versions. 

397 """ 

398 return {pkg: other._packages[pkg] for pkg in other._names - self._names} 

399 

400 def difference(self, other): 

401 """Get packages in symmetric difference of self and another `Packages` 

402 object. 

403 

404 Parameters 

405 ---------- 

406 other : `Packages` 

407 Other packages to compare against. 

408 

409 Returns 

410 ------- 

411 difference : `dict` 

412 Packages in symmetric difference. Keys (type `str`) are package 

413 names; values (type `str`) are their versions. 

414 """ 

415 return {pkg: (self._packages[pkg], other._packages[pkg]) for 

416 pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]} 

417 

418 

419# Register YAML representers 

420 

421def pkg_representer(dumper, data): 

422 """Represent Packages as a simple dict""" 

423 return dumper.represent_mapping("lsst.base.Packages", data._packages, 

424 flow_style=None) 

425 

426 

427yaml.add_representer(Packages, pkg_representer) 

428 

429 

430def pkg_constructor(loader, node): 

431 yield Packages(loader.construct_mapping(node, deep=True)) 

432 

433 

434for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader): 

435 yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)