Coverage for python/lsst/sconsUtils/installation.py: 8%
231 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 21:43 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 21:43 -0800
1"""Builders and path setup for installation targets."""
3__all__ = ("makeProductPath", "determineVersion", "getFingerprint", "setPrefix", "DirectoryInstaller",
4 "SConsUtilsEnvironment")
6import os.path
7import glob
8import re
9import shutil
11import SCons.Script
12from SCons.Script.SConscript import SConsEnvironment
14from .vcs import svn
15from .vcs import hg
16from .vcs import git
18from . import state
19from .utils import memberOf
22class SConsUtilsEnvironment(SConsEnvironment):
23 """Dummy class to make visible the methods injected into the SCons
24 parent environment.
25 """
28def makeProductPath(env, pathFormat):
29 """Return a path to use as the installation directory for a product.
31 Parameters
32 ----------
33 env : `SCons.Environment`
34 The SCons environment.
35 pathFormat : `str`
36 The format string to process.
38 Returns
39 -------
40 formatted : `str`
41 Formatted path string.
42 """
43 pathFormat = re.sub(r"%(\w)", r"%(\1)s", pathFormat)
45 eupsPath = os.environ['PWD']
46 if 'eupsPath' in env and env['eupsPath']:
47 eupsPath = env['eupsPath']
49 return pathFormat % {"P": eupsPath,
50 "f": env['eupsFlavor'],
51 "p": env['eupsProduct'],
52 "v": env['version'],
53 "c": os.environ['PWD']}
56def determineVersion(env, versionString):
57 """Set a version ID from env, or a version control ID string
58 (``$name$`` or ``$HeadURL$``).
60 Parameters
61 ----------
62 env : `SCons.Environment`
63 The SCons environment.
64 versionString : `str`
65 The string containining version information to search if the
66 version can not be found in the environment.
68 Returns
69 -------
70 version : `str`
71 The version.
72 """
73 version = "unknown"
74 if 'version' in env:
75 version = env['version']
76 elif not versionString:
77 version = "unknown"
78 elif re.search(r"^[$]Name:\s+", versionString):
79 # CVS. Extract the tagname
80 version = re.search(r"^[$]Name:\s+([^ $]*)", versionString).group(1)
81 if version == "":
82 version = "cvs"
83 elif re.search(r"^[$]HeadURL:\s+", versionString):
84 # SVN. Guess the tagname from the last part of the directory
85 HeadURL = re.search(r"^[$]HeadURL:\s+(.*)", versionString).group(1)
86 HeadURL = os.path.split(HeadURL)[0]
87 version = svn.guessVersionName(HeadURL)
88 elif versionString.lower() in ("hg", "mercurial"):
89 # Mercurial (hg).
90 version = hg.guessVersionName()
91 elif versionString.lower() in ("git",):
92 # git.
93 version = git.guessVersionName()
94 return version.replace("/", "_")
97def getFingerprint(versionString):
98 """Return a unique fingerprint for a version (e.g. an SHA1);
100 Parameters
101 ----------
102 versionString : `str`
103 A string that might contain version information.
105 Returns
106 -------
107 fingerprint : `str`
108 Unique fingerprint of this version. `None` if unavailable.
109 """
110 if versionString.lower() in ("hg", "mercurial"):
111 fingerprint, modified = hg.guessFingerprint()
112 elif versionString.lower() in ("git",):
113 fingerprint, modified = git.guessFingerprint()
114 else:
115 fingerprint, modified = None, False
117 if fingerprint and modified:
118 fingerprint += " *"
120 return fingerprint
123def setPrefix(env, versionString, eupsProductPath=None):
124 """Set a prefix based on the EUPS_PATH, the product name, and a
125 version string from CVS or SVN.
127 Parameters
128 ----------
129 env : `SCons.Environment`
130 Environment to search.
131 versionString : `str`
132 String that might contain version information.
133 eupsProductPath : `str`, optional
134 Path to the EUPS product.
136 Returns
137 -------
138 prefix : `str`
139 Prefix to use.
140 """
141 try:
142 env['version'] = determineVersion(env, versionString)
143 except RuntimeError as err:
144 env['version'] = "unknown"
145 if (env.installing or env.declaring) and not env['force']:
146 state.log.fail(
147 "%s\nFound problem with version number; update or specify force=True to proceed"
148 % err
149 )
151 if state.env['no_eups']:
152 if 'prefix' in env and env['prefix']:
153 return env['prefix']
154 else:
155 return "/usr/local"
157 if eupsProductPath:
158 eupsPrefix = makeProductPath(env, eupsProductPath)
159 elif 'eupsPath' in env and env['eupsPath']:
160 eupsPrefix = env['eupsPath']
161 else:
162 state.log.fail("Unable to determine eupsPrefix from eupsProductPath or eupsPath")
163 flavor = env['eupsFlavor']
164 if not re.search("/" + flavor + "$", eupsPrefix):
165 eupsPrefix = os.path.join(eupsPrefix, flavor)
166 prodPath = env['eupsProduct']
167 if 'eupsProductPath' in env and env['eupsProductPath']:
168 prodPath = env['eupsProductPath']
169 eupsPrefix = os.path.join(eupsPrefix, prodPath, env["version"])
170 else:
171 eupsPrefix = None
172 if 'prefix' in env:
173 if env['version'] != "unknown" and eupsPrefix and eupsPrefix != env['prefix']:
174 state.log.warn("Ignoring prefix %s from EUPS_PATH" % eupsPrefix)
175 return makeProductPath(env, env['prefix'])
176 elif 'eupsPath' in env and env['eupsPath']:
177 prefix = eupsPrefix
178 else:
179 prefix = "/usr/local"
180 return prefix
183@memberOf(SConsEnvironment)
184def Declare(self, products=None):
185 """Create current and declare targets for products.
187 Parameters
188 ----------
189 products : `list` of `tuple`, optional
190 A list of ``(product, version)`` tuples. If ``product`` is `None`
191 it's taken to be ``self['eupsProduct']``; if version is `None` it's
192 taken to be ``self['version']``.
194 Returns
195 -------
196 acts : `list`
197 Commands to execute.
198 """
200 if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS and not self.GetOption("silent"):
201 state.log.warn("'scons undeclare' is deprecated; please use 'scons declare -c' instead")
203 acts = []
204 if ("declare" in SCons.Script.COMMAND_LINE_TARGETS
205 or "undeclare" in SCons.Script.COMMAND_LINE_TARGETS
206 or ("install" in SCons.Script.COMMAND_LINE_TARGETS and self.GetOption("clean"))
207 or "current" in SCons.Script.COMMAND_LINE_TARGETS):
208 current = []
209 declare = []
210 undeclare = []
212 if not products:
213 products = [None]
215 for prod in products:
216 if not prod or isinstance(prod, str): # i.e. no version
217 product = prod
219 if 'version' in self:
220 version = self['version']
221 else:
222 version = None
223 else:
224 product, version = prod
226 if not product:
227 product = self['eupsProduct']
229 if "EUPS_DIR" in os.environ:
230 self['ENV']['PATH'] += os.pathsep + "%s/bin" % (os.environ["EUPS_DIR"])
231 self["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1")
232 if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS or self.GetOption("clean"):
233 if version:
234 command = "eups undeclare --flavor %s %s %s" % \
235 (self['eupsFlavor'], product, version)
236 if ("current" in SCons.Script.COMMAND_LINE_TARGETS
237 and "declare" not in SCons.Script.COMMAND_LINE_TARGETS):
238 command += " --current"
240 if self.GetOption("clean"):
241 self.Execute(command)
242 else:
243 undeclare += [command]
244 else:
245 state.log.warn("I don't know your version; not undeclaring to eups")
246 else:
247 command = "eups declare --force --flavor %s --root %s" % \
248 (self['eupsFlavor'], self['prefix'])
250 if 'eupsPath' in self:
251 command += " -Z %s" % self['eupsPath']
253 if version:
254 command += " %s %s" % (product, version)
256 current += [command + " --current"]
258 if self.GetOption("tag"):
259 command += " --tag=%s" % self.GetOption("tag")
261 declare += [command]
263 if current:
264 acts += self.Command("current", "", action=current)
265 if declare:
266 if "current" in SCons.Script.COMMAND_LINE_TARGETS:
267 acts += self.Command("declare", "", action="") # current will declare it for us
268 else:
269 acts += self.Command("declare", "", action=declare)
270 if undeclare:
271 acts += self.Command("undeclare", "", action=undeclare)
273 return acts
276class DirectoryInstaller:
277 """SCons Action callable to recursively install a directory.
279 This is separate from the InstallDir function to allow the
280 directory-walking to happen when installation is actually invoked,
281 rather than when the SConscripts are parsed. This still does not ensure
282 that all necessary files are built as prerequisites to installing, but
283 if one explicitly marks the install targets as dependent on the build
284 targets, that should be enough.
286 Parameters
287 ----------
288 ignoreRegex : `str`
289 Regular expression to use to ignore files and directories.
290 recursive : `bool`
291 Control whether to recurse through directories.
292 """
294 def __init__(self, ignoreRegex, recursive):
295 self.ignoreRegex = re.compile(ignoreRegex)
296 self.recursive = recursive
298 def __call__(self, target, source, env):
299 prefix = os.path.abspath(os.path.join(target[0].abspath, ".."))
300 destpath = os.path.join(target[0].abspath)
301 if not os.path.isdir(destpath):
302 state.log.info("Creating directory %s" % destpath)
303 os.makedirs(destpath)
304 for root, dirnames, filenames in os.walk(source[0].path):
305 if not self.recursive:
306 dirnames[:] = []
307 else:
308 dirnames[:] = [d for d in dirnames if d != ".svn"] # ignore .svn tree
309 for dirname in dirnames:
310 destpath = os.path.join(prefix, root, dirname)
311 if not os.path.isdir(destpath):
312 state.log.info("Creating directory %s" % destpath)
313 os.makedirs(destpath)
314 for filename in filenames:
315 if self.ignoreRegex.search(filename):
316 continue
317 destpath = os.path.join(prefix, root)
318 srcpath = os.path.join(root, filename)
319 state.log.info("Copying %s to %s" % (srcpath, destpath))
320 shutil.copy(srcpath, destpath)
321 return 0
324@memberOf(SConsEnvironment)
325def InstallDir(self, prefix, dir, ignoreRegex=r"(~$|\.pyc$|\.os?$)", recursive=True):
326 """Install the directory dir into prefix, ignoring certain files.
328 Parameters
329 ----------
330 prefix : `str`
331 Prefix to use for installation.
332 dir : `str`
333 Directory to install.
334 ignoreRegex : `str`
335 Regular expression to control whether a file is ignored.
336 recursive : `bool`
337 Recurse into directories?
339 Returns
340 -------
341 result : `bool`
342 Was installation successful?
343 """
344 if not self.installing:
345 return []
346 result = self.Command(target=os.path.join(self.Dir(prefix).abspath, dir), source=dir,
347 action=DirectoryInstaller(ignoreRegex, recursive))
348 self.AlwaysBuild(result)
349 return result
352@memberOf(SConsEnvironment)
353def InstallEups(env, dest, files=[], presetup=""):
354 """Install a ups directory, setting absolute versions as appropriate
355 (unless you're installing from the trunk, in which case no versions
356 are expanded).
358 Parameters
359 ----------
360 env : `SCons.Environment`
361 Environment to use.
362 dest : `str`
363 Destination directory.
364 files : `list`, optional
365 List of files to install. Any build/table files present in ``./ups``
366 are automatically added to this list.
367 presetup : `dict`, optional
368 A dictionary with keys product names and values the version that
369 should be installed into the table files, overriding eups
370 expandtable's usual behaviour.
372 Returns
373 -------
374 acts : `list`
375 Commands to execute.
377 Notes
378 -----
379 Sample usage:
381 .. code-block:: python
383 env.InstallEups(os.path.join(env['prefix'], "ups"),
384 presetup={"sconsUtils" : env['version']})
385 """
386 acts = []
387 if not env.installing:
388 return acts
390 if env.GetOption("clean"):
391 state.log.warn("Removing" + dest)
392 shutil.rmtree(dest, ignore_errors=True)
393 else:
394 presetupStr = []
395 for p in presetup:
396 presetupStr += ["--product %s=%s" % (p, presetup[p])]
397 presetup = " ".join(presetupStr)
399 env = env.Clone(ENV=os.environ)
400 #
401 # Add any build/table/cfg files to the desired files
402 #
403 files = [str(f) for f in files] # in case the user used Glob not glob.glob
404 files += glob.glob(os.path.join("ups", "*.build")) + glob.glob(os.path.join("ups", "*.table")) \
405 + glob.glob(os.path.join("ups", "*.cfg")) \
406 + glob.glob(os.path.join("ups", "eupspkg*"))
407 files = list(set(files)) # remove duplicates
409 buildFiles = [f for f in files if re.search(r"\.build$", f)]
410 build_obj = env.Install(dest, buildFiles)
411 acts += build_obj
413 tableFiles = [f for f in files if re.search(r"\.table$", f)]
414 table_obj = env.Install(dest, tableFiles)
415 acts += table_obj
417 eupspkgFiles = [f for f in files if re.search(r"^eupspkg", f)]
418 eupspkg_obj = env.Install(dest, eupspkgFiles)
419 acts += eupspkg_obj
421 miscFiles = [f for f in files if not re.search(r"\.(build|table)$", f)]
422 misc_obj = env.Install(dest, miscFiles)
423 acts += misc_obj
425 try:
426 import eups.lock
428 path = eups.Eups.setEupsPath()
429 if path:
430 locks = eups.lock.takeLocks("setup", path, eups.lock.LOCK_SH) # noqa F841 keep locks active
431 env["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1")
432 except ImportError:
433 state.log.warn("Unable to import eups; not locking")
435 eupsTargets = []
437 for i in build_obj:
438 env.AlwaysBuild(i)
440 cmd = "eups expandbuild -i --version %s " % env['version']
441 if 'baseversion' in env:
442 cmd += " --repoversion %s " % env['baseversion']
443 cmd += str(i)
444 eupsTargets.extend(env.AddPostAction(build_obj, env.Action("%s" % (cmd), cmd)))
446 for i in table_obj:
447 env.AlwaysBuild(i)
449 cmd = "eups expandtable -i -W '^(?!LOCAL:)' " # version doesn't start "LOCAL:"
450 if presetup:
451 cmd += presetup + " "
452 cmd += str(i)
454 act = env.Command("table", "", env.Action("%s" % (cmd), cmd))
455 eupsTargets.extend(act)
456 acts += act
457 env.Depends(act, i)
459 # By declaring that all the Eups operations create a file called
460 # "eups" as a side-effect, even though they don't, SCons knows it
461 # can't run them in parallel (it thinks of the side-effect file as
462 # something like a log, and knows you shouldn't be appending to it
463 # in parallel). When Eups locking is working, we may be able to
464 # remove this.
465 env.SideEffect("eups", eupsTargets)
467 return acts
470@memberOf(SConsEnvironment)
471def InstallLSST(self, prefix, dirs, ignoreRegex=None):
472 """Install directories in the usual LSST way, handling "ups" specially.
474 Parameters
475 ----------
476 prefix : `str`
477 Installation prefix.
478 dirs : `list`
479 Directories to install.
480 ignoreRegex : `str`
481 Regular expression for files and directories to ignore.
483 Returns
484 -------
485 results : `list`
486 Commands to execute.
487 """
488 results = []
489 for d in dirs:
490 # if eups is disabled, the .build & .table files will not be "expanded"
491 if d == "ups" and not state.env['no_eups']:
492 t = self.InstallEups(os.path.join(prefix, "ups"))
493 else:
494 t = self.InstallDir(prefix, d, ignoreRegex=ignoreRegex)
495 self.Depends(t, d)
496 results.extend(t)
497 self.Alias("install", t)
498 self.Clean("install", prefix)
499 return results