Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21""" 

22Determine which packages are being used in the system and their versions 

23""" 

24import os 

25import sys 

26import hashlib 

27import importlib 

28import subprocess 

29import logging 

30import pickle as pickle 

31from collections.abc import Mapping 

32 

33from .versions import getRuntimeVersions 

34 

35log = logging.getLogger(__name__) 

36 

37__all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"] 

38 

39 

40# Packages used at build-time (e.g., header-only) 

41BUILDTIME = set(["boost", "eigen", "tmv"]) 

42 

43# Python modules to attempt to load so we can try to get the version 

44# We do this because the version only appears to be available from python, but we use the library 

45PYTHON = set(["galsim"]) 

46 

47# Packages that don't seem to have a mechanism for reporting the runtime version 

48# We need to guess the version from the environment 

49ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"]) 

50 

51 

52def getVersionFromPythonModule(module): 

53 """Determine the version of a python module. 

54 

55 Parameters 

56 ---------- 

57 module : `module` 

58 Module for which to get version. 

59 

60 Returns 

61 ------- 

62 version : `str` 

63 

64 Raises 

65 ------ 

66 AttributeError 

67 Raised if __version__ attribute is not set. 

68 

69 Notes 

70 ----- 

71 We supplement the version with information from the 

72 ``__dependency_versions__`` (a specific variable set by LSST's 

73 `~lsst.sconsUtils` at build time) only for packages that are typically 

74 used only at build-time. 

75 """ 

76 version = module.__version__ 

77 if hasattr(module, "__dependency_versions__"): 

78 # Add build-time dependencies 

79 deps = module.__dependency_versions__ 

80 buildtime = BUILDTIME & set(deps.keys()) 

81 if buildtime: 

82 version += " with " + " ".join("%s=%s" % (pkg, deps[pkg]) 

83 for pkg in sorted(buildtime)) 

84 return version 

85 

86 

87def getPythonPackages(): 

88 """Get imported python packages and their versions. 

89 

90 Returns 

91 ------- 

92 packages : `dict` 

93 Keys (type `str`) are package names; values (type `str`) are their 

94 versions. 

95 

96 Notes 

97 ----- 

98 We wade through `sys.modules` and attempt to determine the version for each 

99 module. Note, therefore, that we can only report on modules that have 

100 *already* been imported. 

101 

102 We don't include any module for which we cannot determine a version. 

103 """ 

104 # Attempt to import libraries that only report their version in python 

105 for module in PYTHON: 

106 try: 

107 importlib.import_module(module) 

108 except Exception: 

109 pass # It's not available, so don't care 

110 

111 packages = {"python": sys.version} 

112 # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions 

113 moduleNames = list(sys.modules.keys()) 

114 for name in moduleNames: 

115 module = sys.modules[name] 

116 try: 

117 ver = getVersionFromPythonModule(module) 

118 except Exception: 

119 continue # Can't get a version from it, don't care 

120 

121 # Remove "foo.bar.version" in favor of "foo.bar" 

122 # This prevents duplication when the __init__.py includes "from .version import *" 

123 for ending in (".version", "._version"): 

124 if name.endswith(ending): 

125 name = name[:-len(ending)] 

126 if name in packages: 

127 assert ver == packages[name] 

128 elif name in packages: 

129 assert ver == packages[name] 

130 

131 # Use LSST package names instead of python module names 

132 # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time 

133 # versions if the environment reveals that we're not using the packages as-built. 

134 if "lsst" in name: 

135 name = name.replace("lsst.", "").replace(".", "_") 

136 

137 packages[name] = ver 

138 

139 return packages 

140 

141 

142_eups = None # Singleton Eups object 

143 

144 

145def getEnvironmentPackages(): 

146 """Get products and their versions from the environment. 

147 

148 Returns 

149 ------- 

150 packages : `dict` 

151 Keys (type `str`) are product names; values (type `str`) are their 

152 versions. 

153 

154 Notes 

155 ----- 

156 We use EUPS to determine the version of certain products (those that don't 

157 provide a means to determine the version any other way) and to check if 

158 uninstalled packages are being used. We only report the product/version 

159 for these packages. 

160 """ 

161 try: 

162 from eups import Eups 

163 from eups.Product import Product 

164 except ImportError: 

165 log.warning("Unable to import eups, so cannot determine package versions from environment") 

166 return {} 

167 

168 # Cache eups object since creating it can take a while 

169 global _eups 

170 if not _eups: 

171 _eups = Eups() 

172 products = _eups.findProducts(tags=["setup"]) 

173 

174 # Get versions for things we can't determine via runtime mechanisms 

175 # XXX Should we just grab everything we can, rather than just a predetermined set? 

176 packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT} 

177 

178 # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled 

179 # code, so the version could be different than what's being reported by the runtime environment (because 

180 # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils 

181 # probably doesn't check to see if the repo is clean). 

182 for prod in products: 

183 if not prod.version.startswith(Product.LocalVersionPrefix): 

184 continue 

185 ver = prod.version 

186 

187 gitDir = os.path.join(prod.dir, ".git") 

188 if os.path.exists(gitDir): 

189 # get the git revision and an indication if the working copy is clean 

190 revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"] 

191 diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff", 

192 "--patch"] 

193 try: 

194 rev = subprocess.check_output(revCmd).decode().strip() 

195 diff = subprocess.check_output(diffCmd) 

196 except Exception: 

197 ver += "@GIT_ERROR" 

198 else: 

199 ver += "@" + rev 

200 if diff: 

201 ver += "+" + hashlib.md5(diff).hexdigest() 

202 else: 

203 ver += "@NO_GIT" 

204 

205 packages[prod.name] = ver 

206 return packages 

207 

208 

209class Packages: 

210 """A table of packages and their versions. 

211 

212 There are a few different types of packages, and their versions are collected 

213 in different ways: 

214 

215 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from 

216 interrogating the dynamic library 

217 2. Python modules (e.g., afw, numpy; galsim is also in this group even though 

218 we only use it through the library, because no version information is 

219 currently provided through the library): we get their version from the 

220 ``__version__`` module variable. Note that this means that we're only aware 

221 of modules that have already been imported. 

222 3. Other packages provide no run-time accessible version information (e.g., 

223 astrometry_net): we get their version from interrogating the environment. 

224 Currently, that means EUPS; if EUPS is replaced or dropped then we'll need 

225 to consider an alternative means of getting this version information. 

226 4. Local versions of packages (a non-installed EUPS package, selected with 

227 ``setup -r /path/to/package``): we identify these through the environment 

228 (EUPS again) and use as a version the path supplemented with the ``git`` 

229 SHA and, if the git repo isn't clean, an MD5 of the diff. 

230 

231 These package versions are collected and stored in a Packages object, which 

232 provides useful comparison and persistence features. 

233 

234 Example usage: 

235 

236 .. code-block:: python 

237 

238 from lsst.base import Packages 

239 pkgs = Packages.fromSystem() 

240 print("Current packages:", pkgs) 

241 old = Packages.read("/path/to/packages.pickle") 

242 print("Old packages:", old) 

243 print("Missing packages compared to before:", pkgs.missing(old)) 

244 print("Extra packages compared to before:", pkgs.extra(old)) 

245 print("Different packages: ", pkgs.difference(old)) 

246 old.update(pkgs) # Include any new packages in the old 

247 old.write("/path/to/packages.pickle") 

248 

249 Parameters 

250 ---------- 

251 packages : `dict` 

252 A mapping {package: version} where both keys and values are type `str`. 

253 

254 Notes 

255 ----- 

256 This is essentially a wrapper around a dict with some conveniences. 

257 """ 

258 

259 def __init__(self, packages): 

260 assert isinstance(packages, Mapping) 

261 self._packages = packages 

262 self._names = set(packages.keys()) 

263 

264 @classmethod 

265 def fromSystem(cls): 

266 """Construct a `Packages` by examining the system. 

267 

268 Determine packages by examining python's `sys.modules`, runtime 

269 libraries and EUPS. 

270 

271 Returns 

272 ------- 

273 packages : `Packages` 

274 """ 

275 packages = {} 

276 packages.update(getPythonPackages()) 

277 packages.update(getRuntimeVersions()) 

278 packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions 

279 return cls(packages) 

280 

281 @classmethod 

282 def read(cls, filename): 

283 """Read packages from filename. 

284 

285 Parameters 

286 ---------- 

287 filename : `str` 

288 Filename from which to read. 

289 

290 Returns 

291 ------- 

292 packages : `Packages` 

293 """ 

294 with open(filename, "rb") as ff: 

295 return pickle.load(ff) 

296 

297 def write(self, filename): 

298 """Write to file. 

299 

300 Parameters 

301 ---------- 

302 filename : `str` 

303 Filename to which to write. 

304 """ 

305 with open(filename, "wb") as ff: 

306 pickle.dump(self, ff) 

307 

308 def __len__(self): 

309 return len(self._packages) 

310 

311 def __str__(self): 

312 ss = "%s({\n" % self.__class__.__name__ 

313 # Sort alphabetically by module name, for convenience in reading 

314 ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for 

315 prod in sorted(self._names)) 

316 ss += ",\n})" 

317 return ss 

318 

319 def __repr__(self): 

320 return "%s(%s)" % (self.__class__.__name__, repr(self._packages)) 

321 

322 def __contains__(self, pkg): 

323 return pkg in self._packages 

324 

325 def __iter__(self): 

326 return iter(self._packages) 

327 

328 def update(self, other): 

329 """Update packages with contents of another set of packages. 

330 

331 Parameters 

332 ---------- 

333 other : `Packages` 

334 Other packages to merge with self. 

335 

336 Notes 

337 ----- 

338 No check is made to see if we're clobbering anything. 

339 """ 

340 self._packages.update(other._packages) 

341 self._names.update(other._names) 

342 

343 def extra(self, other): 

344 """Get packages in self but not in another `Packages` object. 

345 

346 Parameters 

347 ---------- 

348 other : `Packages` 

349 Other packages to compare against. 

350 

351 Returns 

352 ------- 

353 extra : `dict` 

354 Extra packages. Keys (type `str`) are package names; values 

355 (type `str`) are their versions. 

356 """ 

357 return {pkg: self._packages[pkg] for pkg in self._names - other._names} 

358 

359 def missing(self, other): 

360 """Get packages in another `Packages` object but missing from self. 

361 

362 Parameters 

363 ---------- 

364 other : `Packages` 

365 Other packages to compare against. 

366 

367 Returns 

368 ------- 

369 missing : `dict` 

370 Missing packages. Keys (type `str`) are package names; values 

371 (type `str`) are their versions. 

372 """ 

373 return {pkg: other._packages[pkg] for pkg in other._names - self._names} 

374 

375 def difference(self, other): 

376 """Get packages in symmetric difference of self and another `Packages` 

377 object. 

378 

379 Parameters 

380 ---------- 

381 other : `Packages` 

382 Other packages to compare against. 

383 

384 Returns 

385 ------- 

386 difference : `dict` 

387 Packages in symmetric difference. Keys (type `str`) are package 

388 names; values (type `str`) are their versions. 

389 """ 

390 return {pkg: (self._packages[pkg], other._packages[pkg]) for 

391 pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}