Coverage for python/lsst/sconsUtils/installation.py: 8%

231 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-24 23:29 -0700

1"""Builders and path setup for installation targets.""" 

2 

3__all__ = ("makeProductPath", "determineVersion", "getFingerprint", "setPrefix", "DirectoryInstaller", 

4 "SConsUtilsEnvironment") 

5 

6import os.path 

7import glob 

8import re 

9import shutil 

10 

11import SCons.Script 

12from SCons.Script.SConscript import SConsEnvironment 

13 

14from .vcs import svn 

15from .vcs import hg 

16from .vcs import git 

17 

18from . import state 

19from .utils import memberOf 

20 

21 

22class SConsUtilsEnvironment(SConsEnvironment): 

23 """Dummy class to make visible the methods injected into the SCons 

24 parent environment. 

25 """ 

26 

27 

28def makeProductPath(env, pathFormat): 

29 """Return a path to use as the installation directory for a product. 

30 

31 Parameters 

32 ---------- 

33 env : `SCons.Environment` 

34 The SCons environment. 

35 pathFormat : `str` 

36 The format string to process. 

37 

38 Returns 

39 ------- 

40 formatted : `str` 

41 Formatted path string. 

42 """ 

43 pathFormat = re.sub(r"%(\w)", r"%(\1)s", pathFormat) 

44 

45 eupsPath = os.environ['PWD'] 

46 if 'eupsPath' in env and env['eupsPath']: 

47 eupsPath = env['eupsPath'] 

48 

49 return pathFormat % {"P": eupsPath, 

50 "f": env['eupsFlavor'], 

51 "p": env['eupsProduct'], 

52 "v": env['version'], 

53 "c": os.environ['PWD']} 

54 

55 

56def determineVersion(env, versionString): 

57 """Set a version ID from env, or a version control ID string 

58 (``$name$`` or ``$HeadURL$``). 

59 

60 Parameters 

61 ---------- 

62 env : `SCons.Environment` 

63 The SCons environment. 

64 versionString : `str` 

65 The string containining version information to search if the 

66 version can not be found in the environment. 

67 

68 Returns 

69 ------- 

70 version : `str` 

71 The version. 

72 """ 

73 version = "unknown" 

74 if 'version' in env: 

75 version = env['version'] 

76 elif not versionString: 

77 version = "unknown" 

78 elif re.search(r"^[$]Name:\s+", versionString): 

79 # CVS. Extract the tagname 

80 version = re.search(r"^[$]Name:\s+([^ $]*)", versionString).group(1) 

81 if version == "": 

82 version = "cvs" 

83 elif re.search(r"^[$]HeadURL:\s+", versionString): 

84 # SVN. Guess the tagname from the last part of the directory 

85 HeadURL = re.search(r"^[$]HeadURL:\s+(.*)", versionString).group(1) 

86 HeadURL = os.path.split(HeadURL)[0] 

87 version = svn.guessVersionName(HeadURL) 

88 elif versionString.lower() in ("hg", "mercurial"): 

89 # Mercurial (hg). 

90 version = hg.guessVersionName() 

91 elif versionString.lower() in ("git",): 

92 # git. 

93 version = git.guessVersionName() 

94 return version.replace("/", "_") 

95 

96 

97def getFingerprint(versionString): 

98 """Return a unique fingerprint for a version (e.g. an SHA1); 

99 

100 Parameters 

101 ---------- 

102 versionString : `str` 

103 A string that might contain version information. 

104 

105 Returns 

106 ------- 

107 fingerprint : `str` 

108 Unique fingerprint of this version. `None` if unavailable. 

109 """ 

110 if versionString.lower() in ("hg", "mercurial"): 

111 fingerprint, modified = hg.guessFingerprint() 

112 elif versionString.lower() in ("git",): 

113 fingerprint, modified = git.guessFingerprint() 

114 else: 

115 fingerprint, modified = None, False 

116 

117 if fingerprint and modified: 

118 fingerprint += " *" 

119 

120 return fingerprint 

121 

122 

123def setPrefix(env, versionString, eupsProductPath=None): 

124 """Set a prefix based on the EUPS_PATH, the product name, and a 

125 version string from CVS or SVN. 

126 

127 Parameters 

128 ---------- 

129 env : `SCons.Environment` 

130 Environment to search. 

131 versionString : `str` 

132 String that might contain version information. 

133 eupsProductPath : `str`, optional 

134 Path to the EUPS product. 

135 

136 Returns 

137 ------- 

138 prefix : `str` 

139 Prefix to use. 

140 """ 

141 try: 

142 env['version'] = determineVersion(env, versionString) 

143 except RuntimeError as err: 

144 env['version'] = "unknown" 

145 if (env.installing or env.declaring) and not env['force']: 

146 state.log.fail( 

147 "%s\nFound problem with version number; update or specify force=True to proceed" 

148 % err 

149 ) 

150 

151 if state.env['no_eups']: 

152 if 'prefix' in env and env['prefix']: 

153 return env['prefix'] 

154 else: 

155 return "/usr/local" 

156 

157 if eupsProductPath: 

158 eupsPrefix = makeProductPath(env, eupsProductPath) 

159 elif 'eupsPath' in env and env['eupsPath']: 

160 eupsPrefix = env['eupsPath'] 

161 else: 

162 state.log.fail("Unable to determine eupsPrefix from eupsProductPath or eupsPath") 

163 flavor = env['eupsFlavor'] 

164 if not re.search("/" + flavor + "$", eupsPrefix): 

165 eupsPrefix = os.path.join(eupsPrefix, flavor) 

166 prodPath = env['eupsProduct'] 

167 if 'eupsProductPath' in env and env['eupsProductPath']: 

168 prodPath = env['eupsProductPath'] 

169 eupsPrefix = os.path.join(eupsPrefix, prodPath, env["version"]) 

170 else: 

171 eupsPrefix = None 

172 if 'prefix' in env: 

173 if env['version'] != "unknown" and eupsPrefix and eupsPrefix != env['prefix']: 

174 state.log.warn("Ignoring prefix %s from EUPS_PATH" % eupsPrefix) 

175 return makeProductPath(env, env['prefix']) 

176 elif 'eupsPath' in env and env['eupsPath']: 

177 prefix = eupsPrefix 

178 else: 

179 prefix = "/usr/local" 

180 return prefix 

181 

182 

183@memberOf(SConsEnvironment) 

184def Declare(self, products=None): 

185 """Create current and declare targets for products. 

186 

187 Parameters 

188 ---------- 

189 products : `list` of `tuple`, optional 

190 A list of ``(product, version)`` tuples. If ``product`` is `None` 

191 it's taken to be ``self['eupsProduct']``; if version is `None` it's 

192 taken to be ``self['version']``. 

193 

194 Returns 

195 ------- 

196 acts : `list` 

197 Commands to execute. 

198 """ 

199 

200 if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS and not self.GetOption("silent"): 

201 state.log.warn("'scons undeclare' is deprecated; please use 'scons declare -c' instead") 

202 

203 acts = [] 

204 if ("declare" in SCons.Script.COMMAND_LINE_TARGETS 

205 or "undeclare" in SCons.Script.COMMAND_LINE_TARGETS 

206 or ("install" in SCons.Script.COMMAND_LINE_TARGETS and self.GetOption("clean")) 

207 or "current" in SCons.Script.COMMAND_LINE_TARGETS): 

208 current = [] 

209 declare = [] 

210 undeclare = [] 

211 

212 if not products: 

213 products = [None] 

214 

215 for prod in products: 

216 if not prod or isinstance(prod, str): # i.e. no version 

217 product = prod 

218 

219 if 'version' in self: 

220 version = self['version'] 

221 else: 

222 version = None 

223 else: 

224 product, version = prod 

225 

226 if not product: 

227 product = self['eupsProduct'] 

228 

229 if "EUPS_DIR" in os.environ: 

230 self['ENV']['PATH'] += os.pathsep + "%s/bin" % (os.environ["EUPS_DIR"]) 

231 self["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1") 

232 if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS or self.GetOption("clean"): 

233 if version: 

234 command = "eups undeclare --flavor %s %s %s" % \ 

235 (self['eupsFlavor'], product, version) 

236 if ("current" in SCons.Script.COMMAND_LINE_TARGETS 

237 and "declare" not in SCons.Script.COMMAND_LINE_TARGETS): 

238 command += " --current" 

239 

240 if self.GetOption("clean"): 

241 self.Execute(command) 

242 else: 

243 undeclare += [command] 

244 else: 

245 state.log.warn("I don't know your version; not undeclaring to eups") 

246 else: 

247 command = "eups declare --force --flavor %s --root %s" % \ 

248 (self['eupsFlavor'], self['prefix']) 

249 

250 if 'eupsPath' in self: 

251 command += " -Z %s" % self['eupsPath'] 

252 

253 if version: 

254 command += " %s %s" % (product, version) 

255 

256 current += [command + " --current"] 

257 

258 if self.GetOption("tag"): 

259 command += " --tag=%s" % self.GetOption("tag") 

260 

261 declare += [command] 

262 

263 if current: 

264 acts += self.Command("current", "", action=current) 

265 if declare: 

266 if "current" in SCons.Script.COMMAND_LINE_TARGETS: 

267 acts += self.Command("declare", "", action="") # current will declare it for us 

268 else: 

269 acts += self.Command("declare", "", action=declare) 

270 if undeclare: 

271 acts += self.Command("undeclare", "", action=undeclare) 

272 

273 return acts 

274 

275 

276class DirectoryInstaller: 

277 """SCons Action callable to recursively install a directory. 

278 

279 This is separate from the InstallDir function to allow the 

280 directory-walking to happen when installation is actually invoked, 

281 rather than when the SConscripts are parsed. This still does not ensure 

282 that all necessary files are built as prerequisites to installing, but 

283 if one explicitly marks the install targets as dependent on the build 

284 targets, that should be enough. 

285 

286 Parameters 

287 ---------- 

288 ignoreRegex : `str` 

289 Regular expression to use to ignore files and directories. 

290 recursive : `bool` 

291 Control whether to recurse through directories. 

292 """ 

293 

294 def __init__(self, ignoreRegex, recursive): 

295 self.ignoreRegex = re.compile(ignoreRegex) 

296 self.recursive = recursive 

297 

298 def __call__(self, target, source, env): 

299 prefix = os.path.abspath(os.path.join(target[0].abspath, "..")) 

300 destpath = os.path.join(target[0].abspath) 

301 if not os.path.isdir(destpath): 

302 state.log.info("Creating directory %s" % destpath) 

303 os.makedirs(destpath) 

304 for root, dirnames, filenames in os.walk(source[0].path): 

305 if not self.recursive: 

306 dirnames[:] = [] 

307 else: 

308 dirnames[:] = [d for d in dirnames if d != ".svn"] # ignore .svn tree 

309 for dirname in dirnames: 

310 destpath = os.path.join(prefix, root, dirname) 

311 if not os.path.isdir(destpath): 

312 state.log.info("Creating directory %s" % destpath) 

313 os.makedirs(destpath) 

314 for filename in filenames: 

315 if self.ignoreRegex.search(filename): 

316 continue 

317 destpath = os.path.join(prefix, root) 

318 srcpath = os.path.join(root, filename) 

319 state.log.info("Copying %s to %s" % (srcpath, destpath)) 

320 shutil.copy(srcpath, destpath) 

321 return 0 

322 

323 

324@memberOf(SConsEnvironment) 

325def InstallDir(self, prefix, dir, ignoreRegex=r"(~$|\.pyc$|\.os?$)", recursive=True): 

326 """Install the directory dir into prefix, ignoring certain files. 

327 

328 Parameters 

329 ---------- 

330 prefix : `str` 

331 Prefix to use for installation. 

332 dir : `str` 

333 Directory to install. 

334 ignoreRegex : `str` 

335 Regular expression to control whether a file is ignored. 

336 recursive : `bool` 

337 Recurse into directories? 

338 

339 Returns 

340 ------- 

341 result : `bool` 

342 Was installation successful? 

343 """ 

344 if not self.installing: 

345 return [] 

346 result = self.Command(target=os.path.join(self.Dir(prefix).abspath, dir), source=dir, 

347 action=DirectoryInstaller(ignoreRegex, recursive)) 

348 self.AlwaysBuild(result) 

349 return result 

350 

351 

352@memberOf(SConsEnvironment) 

353def InstallEups(env, dest, files=[], presetup=""): 

354 """Install a ups directory, setting absolute versions as appropriate 

355 (unless you're installing from the trunk, in which case no versions 

356 are expanded). 

357 

358 Parameters 

359 ---------- 

360 env : `SCons.Environment` 

361 Environment to use. 

362 dest : `str` 

363 Destination directory. 

364 files : `list`, optional 

365 List of files to install. Any build/table files present in ``./ups`` 

366 are automatically added to this list. 

367 presetup : `dict`, optional 

368 A dictionary with keys product names and values the version that 

369 should be installed into the table files, overriding eups 

370 expandtable's usual behaviour. 

371 

372 Returns 

373 ------- 

374 acts : `list` 

375 Commands to execute. 

376 

377 Notes 

378 ----- 

379 Sample usage: 

380 

381 .. code-block:: python 

382 

383 env.InstallEups(os.path.join(env['prefix'], "ups"), 

384 presetup={"sconsUtils" : env['version']}) 

385 """ 

386 acts = [] 

387 if not env.installing: 

388 return acts 

389 

390 if env.GetOption("clean"): 

391 state.log.warn("Removing" + dest) 

392 shutil.rmtree(dest, ignore_errors=True) 

393 else: 

394 presetupStr = [] 

395 for p in presetup: 

396 presetupStr += ["--product %s=%s" % (p, presetup[p])] 

397 presetup = " ".join(presetupStr) 

398 

399 env = env.Clone(ENV=os.environ) 

400 # 

401 # Add any build/table/cfg files to the desired files 

402 # 

403 files = [str(f) for f in files] # in case the user used Glob not glob.glob 

404 files += glob.glob(os.path.join("ups", "*.build")) + glob.glob(os.path.join("ups", "*.table")) \ 

405 + glob.glob(os.path.join("ups", "*.cfg")) \ 

406 + glob.glob(os.path.join("ups", "eupspkg*")) 

407 files = list(set(files)) # remove duplicates 

408 

409 buildFiles = [f for f in files if re.search(r"\.build$", f)] 

410 build_obj = env.Install(dest, buildFiles) 

411 acts += build_obj 

412 

413 tableFiles = [f for f in files if re.search(r"\.table$", f)] 

414 table_obj = env.Install(dest, tableFiles) 

415 acts += table_obj 

416 

417 eupspkgFiles = [f for f in files if re.search(r"^eupspkg", f)] 

418 eupspkg_obj = env.Install(dest, eupspkgFiles) 

419 acts += eupspkg_obj 

420 

421 miscFiles = [f for f in files if not re.search(r"\.(build|table)$", f)] 

422 misc_obj = env.Install(dest, miscFiles) 

423 acts += misc_obj 

424 

425 try: 

426 import eups.lock 

427 

428 path = eups.Eups.setEupsPath() 

429 if path: 

430 locks = eups.lock.takeLocks("setup", path, eups.lock.LOCK_SH) # noqa F841 keep locks active 

431 env["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1") 

432 except ImportError: 

433 state.log.warn("Unable to import eups; not locking") 

434 

435 eupsTargets = [] 

436 

437 for i in build_obj: 

438 env.AlwaysBuild(i) 

439 

440 cmd = "eups expandbuild -i --version %s " % env['version'] 

441 if 'baseversion' in env: 

442 cmd += " --repoversion %s " % env['baseversion'] 

443 cmd += str(i) 

444 eupsTargets.extend(env.AddPostAction(build_obj, env.Action("%s" % (cmd), cmd))) 

445 

446 for i in table_obj: 

447 env.AlwaysBuild(i) 

448 

449 cmd = "eups expandtable -i -W '^(?!LOCAL:)' " # version doesn't start "LOCAL:" 

450 if presetup: 

451 cmd += presetup + " " 

452 cmd += str(i) 

453 

454 act = env.Command("table", "", env.Action("%s" % (cmd), cmd)) 

455 eupsTargets.extend(act) 

456 acts += act 

457 env.Depends(act, i) 

458 

459 # By declaring that all the Eups operations create a file called 

460 # "eups" as a side-effect, even though they don't, SCons knows it 

461 # can't run them in parallel (it thinks of the side-effect file as 

462 # something like a log, and knows you shouldn't be appending to it 

463 # in parallel). When Eups locking is working, we may be able to 

464 # remove this. 

465 env.SideEffect("eups", eupsTargets) 

466 

467 return acts 

468 

469 

470@memberOf(SConsEnvironment) 

471def InstallLSST(self, prefix, dirs, ignoreRegex=None): 

472 """Install directories in the usual LSST way, handling "ups" specially. 

473 

474 Parameters 

475 ---------- 

476 prefix : `str` 

477 Installation prefix. 

478 dirs : `list` 

479 Directories to install. 

480 ignoreRegex : `str` 

481 Regular expression for files and directories to ignore. 

482 

483 Returns 

484 ------- 

485 results : `list` 

486 Commands to execute. 

487 """ 

488 results = [] 

489 for d in dirs: 

490 # if eups is disabled, the .build & .table files will not be "expanded" 

491 if d == "ups" and not state.env['no_eups']: 

492 t = self.InstallEups(os.path.join(prefix, "ups")) 

493 else: 

494 t = self.InstallDir(prefix, d, ignoreRegex=ignoreRegex) 

495 self.Depends(t, d) 

496 results.extend(t) 

497 self.Alias("install", t) 

498 self.Clean("install", prefix) 

499 return results