Coverage for python/lsst/sconsUtils/builders.py: 7%
324 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-01 00:45 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-01 00:45 -0700
1"""Extra builders and methods to be injected into the SConsEnvironment class.
2"""
4__all__ = ("filesToTag", "DoxygenBuilder")
6import os
7import re
8import fnmatch
9import pipes
11import SCons.Script
12from SCons.Script.SConscript import SConsEnvironment
14from .utils import memberOf
15from .installation import determineVersion, getFingerprint
16from . import state
19@memberOf(SConsEnvironment)
20def SharedLibraryIncomplete(self, target, source, **keywords):
21 """Like SharedLibrary, but don't insist that all symbols are resolved.
22 """
23 myenv = self.Clone()
24 if myenv['PLATFORM'] == 'darwin':
25 myenv['SHLINKFLAGS'] += ["-undefined", "dynamic_lookup",
26 "-headerpad_max_install_names"]
27 return myenv.SharedLibrary(target, source, **keywords)
30@memberOf(SConsEnvironment)
31def Pybind11LoadableModule(self, target, source, **keywords):
32 """Like LoadableModule, but don't insist that all symbols are resolved, and
33 set some pybind11-specific flags.
34 """
35 myenv = self.Clone()
36 myenv.Append(CCFLAGS=["-fvisibility=hidden"])
37 if myenv['PLATFORM'] == 'darwin':
38 myenv.Append(LDMODULEFLAGS=["-undefined", "dynamic_lookup",
39 "-headerpad_max_install_names"])
40 return myenv.LoadableModule(target, source, **keywords)
43@memberOf(SConsEnvironment)
44def SourcesForSharedLibrary(self, files):
45 """Prepare the list of files to be passed to a SharedLibrary constructor.
47 Parameters
48 ----------
49 files :
50 List of files to be processed.
52 Returns
53 -------
54 objs : `list`
55 Object files.
57 Notes
58 -----
59 In particular, ensure that any files listed in ``env.NoOptFiles`` (set by
60 the command line option ``noOptFile="file1 file2"``) are built without
61 optimisation and files listed in ``env.optFiles`` are built with
62 optimisation.
64 The usage pattern in an SConscript file is:
66 .. code-block:: python
68 ccFiles = env.SourcesForSharedLibrary(Glob("../src/*/*.cc"))
69 env.SharedLibrary('afw', ccFiles, LIBS=env.getLibs("self")))
71 This is automatically used by
72 `lsst.sconsUtils.scripts.BasicSConscript.lib()`.
73 """
75 files = [SCons.Script.File(file) for file in files]
77 if not (self.get("optFiles") or self.get("noOptFiles")):
78 objs = [self.SharedObject(ccFile) for ccFile in sorted(state.env.Flatten(files), key=str)]
79 return objs
81 if self.get("optFiles"):
82 optFiles = self["optFiles"].replace(".", r"\.") # it'll be used in an RE
83 optFiles = SCons.Script.Split(optFiles.replace(",", " "))
84 optFilesRe = "/(%s)$" % "|".join(optFiles)
85 else:
86 optFilesRe = None
88 if self.get("noOptFiles"):
89 noOptFiles = self["noOptFiles"].replace(".", r"\.") # it'll be used in an RE
90 noOptFiles = SCons.Script.Split(noOptFiles.replace(",", " "))
91 noOptFilesRe = "/(%s)$" % "|".join(noOptFiles)
92 else:
93 noOptFilesRe = None
95 if self.get("opt"):
96 opt = int(self["opt"])
97 else:
98 opt = 0
100 if opt == 0:
101 opt = 3
103 CCFLAGS_OPT = re.sub(r"-O(\d|s)\s*", "-O%d " % opt, " ".join(self["CCFLAGS"]))
104 CCFLAGS_NOOPT = re.sub(r"-O(\d|s)\s*", "-O0 ", " ".join(self["CCFLAGS"])) # remove -O flags from CCFLAGS
106 objs = []
107 for ccFile in files:
108 if optFilesRe and re.search(optFilesRe, ccFile.abspath):
109 obj = self.SharedObject(ccFile, CCFLAGS=CCFLAGS_OPT)
110 elif noOptFilesRe and re.search(noOptFilesRe, ccFile.abspath):
111 obj = self.SharedObject(ccFile, CCFLAGS=CCFLAGS_NOOPT)
112 else:
113 obj = self.SharedObject(ccFile)
114 objs.append(obj)
116 objs = sorted(state.env.Flatten(objs), key=str)
117 return objs
120def filesToTag(root=None, fileRegex=None, ignoreDirs=None):
121 """Return a list of files that need to be scanned for tags, starting at
122 directory root.
124 Parameters
125 ----------
126 root : `str`, optional
127 Directory root to search.
128 fileRegex : `str`, optional
129 Matching regular expression for files.
130 ignoreDirs : `list`
131 List of directories to ignore when searching.
133 Returns
134 -------
135 files : `list`
136 List of matching files.
138 Notes
139 -----
140 These tags are for advanced Emacs users, and should not be confused with
141 SVN tags or Doxygen tags.
143 Files are chosen if they match fileRegex; toplevel directories in list
144 ignoreDirs are ignored.
145 This routine won't do anything unless you specified a "TAGS" target.
146 """
148 if root is None:
149 root = "."
150 if fileRegex is None:
151 fileRegex = r"^[a-zA-Z0-9_].*\.(cc|h(pp)?|py)$"
152 if ignoreDirs is None:
153 ignoreDirs = ["examples", "tests"]
155 if "TAGS" not in SCons.Script.COMMAND_LINE_TARGETS:
156 return []
158 files = []
159 for dirpath, dirnames, filenames in os.walk(root):
160 if dirpath == ".":
161 dirnames[:] = [d for d in dirnames if not re.search(r"^(%s)$" % "|".join(ignoreDirs), d)]
163 dirnames[:] = [d for d in dirnames if not re.search(r"^(\.svn)$", d)] # ignore .svn tree
164 #
165 # List of possible files to tag, but there's some cleanup required
166 # for machine-generated files
167 #
168 candidates = [f for f in filenames if re.search(fileRegex, f)]
169 #
170 # Remove files generated by swig
171 #
172 for swigFile in [f for f in filenames if f.endswith(".i")]:
173 name = os.path.splitext(swigFile)[0]
174 candidates = [f for f in candidates if not re.search(r"%s(_wrap\.cc?|\.py)$" % name, f)]
176 files += [os.path.join(dirpath, f) for f in candidates]
178 return files
181@memberOf(SConsEnvironment)
182def BuildETags(env, root=None, fileRegex=None, ignoreDirs=None):
183 """Build Emacs tags (see man etags for more information).
185 Parameters
186 ----------
187 env : `SCons.Environment`
188 Environment to use to run ``etags`` command.
189 root : `str`, optional
190 Directory to begin search.
191 fileRegex : `str`
192 Regular expression to match files.
193 ignoreDirs : `list`
194 List of directories to ignore.
196 Notes
197 -----
198 Files are chosen if they match fileRegex; toplevel directories in list
199 ignoreDirs are ignored. This routine won't do anything unless you
200 specified a "TAGS" target."""
202 toTag = filesToTag(root, fileRegex, ignoreDirs)
203 if toTag:
204 return env.Command("TAGS", toTag, "etags -o $TARGET $SOURCES")
207@memberOf(SConsEnvironment)
208def CleanTree(self, filePatterns, dirPatterns="", directory=".", verbose=False):
209 """Remove files matching the argument list starting at directory
210 when scons is invoked with -c/--clean and no explicit targets are listed.
212 Parameters
213 ----------
214 filePatterns : `str`
215 Glob to match for files to be deleted.
216 dirPatterns : `str`, optional
217 Specification of directories to be removed.
218 directory : `str`, optional
219 Directory to clean.
220 verbose : `bool`, optional
221 If `True` print each filename after deleting it.
223 Notes
224 -----
225 Can be run as:
227 .. code-block:: python
229 env.CleanTree(r"*~ core")
230 """
232 def genFindCommand(patterns, directory, verbose, filesOnly):
233 # Generate find command to clean up (find-glob) patterns, either files
234 # or directories.
235 expr = ""
236 for pattern in SCons.Script.Split(patterns):
237 if expr != "":
238 expr += " -o "
239 # Quote unquoted * and [
240 expr += "-name %s" % re.sub(r"(^|[^\\])([\[*])", r"\1\\\2", pattern)
241 if filesOnly:
242 expr += " -type f"
243 else:
244 expr += " -type d -prune"
246 command = "find " + directory
247 # Don't look into .svn or .git directories to save time.
248 command += r" \( -name .svn -prune -o -name .git -prune -o -name \* \) "
249 command += r" \( " + expr + r" \)"
250 if filesOnly:
251 command += r" -exec rm -f {} \;"
252 else:
253 command += r" -exec rm -rf {} \;"
254 if verbose:
255 command += " -print"
256 return command
258 action = genFindCommand(filePatterns, directory, verbose, filesOnly=True)
260 # Clean up scons files --- users want to be able to say scons -c and get a
261 # clean copy.
262 # We can't delete .sconsign.dblite if we use "scons clean" instead of
263 # "scons --clean", so the former is no longer supported.
264 action += " ; rm -rf .sconf_temp .sconsign.dblite .sconsign.tmp config.log"
266 if dirPatterns != "":
267 action += " ; "
268 action += genFindCommand(dirPatterns, directory, verbose, filesOnly=False)
269 # Do we actually want to clean up? We don't if the command is e.g.
270 # "scons -c install"
271 if "clean" in SCons.Script.COMMAND_LINE_TARGETS:
272 state.log.fail("'scons clean' is no longer supported; please use 'scons --clean'.")
273 elif not SCons.Script.COMMAND_LINE_TARGETS and self.GetOption("clean"):
274 self.Execute(self.Action([action]))
277@memberOf(SConsEnvironment)
278def ProductDir(env, product):
279 """Return the product directory.
281 Parameters
282 ----------
283 product : `str`
284 The EUPS product name.
286 Returns
287 -------
288 dir : `str`
289 The product directory. `None` if the product is not known.
290 """
291 from . import eupsForScons
292 global _productDirs
293 try:
294 _productDirs
295 except Exception:
296 try:
297 _productDirs = eupsForScons.productDir(eupsenv=eupsForScons.getEups())
298 except TypeError: # old version of eups (pre r18588)
299 _productDirs = None
300 if _productDirs:
301 pdir = _productDirs.get(product)
302 else:
303 pdir = eupsForScons.productDir(product)
304 if pdir == "none":
305 pdir = None
306 return pdir
309class DoxygenBuilder:
310 """A callable to be used as an SCons Action to run Doxygen.
312 This should only be used by the env.Doxygen pseudo-builder method.
313 """
315 def __init__(self, **kw):
316 self.__dict__.update(kw)
317 self.results = []
318 self.sources = []
319 self.targets = []
320 self.useTags = list(SCons.Script.File(item).abspath for item in self.useTags)
321 self.inputs = list(SCons.Script.Entry(item).abspath for item in self.inputs)
322 self.excludes = list(SCons.Script.Entry(item).abspath for item in self.excludes)
323 self.outputPaths = list(SCons.Script.Dir(item) for item in self.outputs)
325 def __call__(self, env, config):
326 self.findSources()
327 self.findTargets()
328 inConfigNode = SCons.Script.File(config)
329 outConfigName, ext = os.path.splitext(inConfigNode.abspath)
330 outConfigNode = SCons.Script.File(outConfigName)
331 if self.makeTag:
332 tagNode = SCons.Script.File(self.makeTag)
333 self.makeTag = tagNode.abspath
334 self.targets.append(tagNode)
335 config = env.Command(target=outConfigNode, source=inConfigNode if os.path.exists(config) else None,
336 action=self.buildConfig)
337 env.AlwaysBuild(config)
338 doc = env.Command(target=self.targets, source=self.sources,
339 action="doxygen %s" % pipes.quote(outConfigNode.abspath))
340 for path in self.outputPaths:
341 env.Clean(doc, path)
342 env.Depends(doc, config)
343 self.results.extend(config)
344 self.results.extend(doc)
345 return self.results
347 def findSources(self):
348 for path in self.inputs:
349 if os.path.isdir(path):
350 for root, dirs, files in os.walk(path):
351 if os.path.abspath(root) in self.excludes:
352 dirs[:] = []
353 continue
354 if not self.recursive:
355 dirs[:] = []
356 else:
357 toKeep = []
358 for relDir in dirs:
359 if relDir.startswith("."):
360 continue
361 absDir = os.path.abspath(os.path.join(root, relDir))
362 if absDir not in self.excludes:
363 toKeep.append(relDir)
364 dirs[:] = toKeep
365 if self.excludeSwig:
366 for relFile in files:
367 base, ext = os.path.splitext(relFile)
368 if ext == ".i":
369 self.excludes.append(os.path.join(root, base + ".py"))
370 self.excludes.append(os.path.join(root, base + "_wrap.cc"))
371 for relFile in files:
372 absFile = os.path.abspath(os.path.join(root, relFile))
373 if absFile in self.excludes:
374 continue
375 for pattern in self.patterns:
376 if fnmatch.fnmatch(relFile, pattern):
377 self.sources.append(SCons.Script.File(absFile))
378 break
379 elif os.path.isfile(path):
380 self.sources.append(SCons.Script.File(path))
382 def findTargets(self):
383 for item in self.outputs:
384 self.targets.append(SCons.Script.Dir(item))
386 def buildConfig(self, target, source, env):
387 outConfigFile = open(target[0].abspath, "w")
389 # Need a routine to quote paths that contain spaces
390 # but can not use pipes.quote because it has to be
391 # a double quote for doxygen.conf
392 # Do not quote a string if it is already quoted
393 # Also have a version that quotes each item in a sequence and generates
394 # the final quoted entry.
395 def _quote_path(path):
396 if " " in path and not path.startswith('"') and not path.endswith('"'):
397 return '"{}"'.format(path)
398 return path
400 def _quote_paths(pathList):
401 return " ".join(_quote_path(p) for p in pathList)
403 docPaths = []
404 incFiles = []
405 for incPath in self.includes:
406 docDir, incFile = os.path.split(incPath)
407 docPaths.append('"%s"' % docDir)
408 incFiles.append('"%s"' % incFile)
409 self.sources.append(SCons.Script.File(incPath))
410 if docPaths:
411 outConfigFile.write('@INCLUDE_PATH = %s\n' % _quote_paths(docPaths))
412 for incFile in incFiles:
413 outConfigFile.write('@INCLUDE = %s\n' % _quote_path(incFile))
415 for tagPath in self.useTags:
416 docDir, tagFile = os.path.split(tagPath)
417 htmlDir = os.path.join(docDir, "html")
418 outConfigFile.write('TAGFILES += "%s=%s"\n' % (tagPath, htmlDir))
419 self.sources.append(SCons.Script.Dir(docDir))
420 if self.projectName is not None:
421 outConfigFile.write("PROJECT_NAME = %s\n" % self.projectName)
422 if self.projectNumber is not None:
423 outConfigFile.write("PROJECT_NUMBER = %s\n" % self.projectNumber)
424 outConfigFile.write("INPUT = %s\n" % _quote_paths(self.inputs))
425 outConfigFile.write("EXCLUDE = %s\n" % _quote_paths(self.excludes))
426 outConfigFile.write("FILE_PATTERNS = %s\n" % " ".join(self.patterns))
427 outConfigFile.write("RECURSIVE = YES\n" if self.recursive else "RECURSIVE = NO\n")
428 allOutputs = set(("html", "latex", "man", "rtf", "xml"))
429 for output, path in zip(self.outputs, self.outputPaths):
430 try:
431 allOutputs.remove(output.lower())
432 except Exception:
433 state.log.fail("Unknown Doxygen output format '%s'." % output)
434 state.log.finish()
435 outConfigFile.write("GENERATE_%s = YES\n" % output.upper())
436 outConfigFile.write("%s_OUTPUT = %s\n" % (output.upper(), _quote_path(path.abspath)))
437 for output in allOutputs:
438 outConfigFile.write("GENERATE_%s = NO\n" % output.upper())
439 if self.makeTag is not None:
440 outConfigFile.write("GENERATE_TAGFILE = %s\n" % _quote_path(self.makeTag))
441 #
442 # Append the local overrides (usually doxygen.conf.in)
443 #
444 if len(source) > 0:
445 with open(source[0].abspath, "r") as inConfigFile:
446 outConfigFile.write(inConfigFile.read())
448 outConfigFile.close()
451@memberOf(SConsEnvironment)
452def Doxygen(self, config, **kwargs):
453 """Generate a Doxygen config file and run Doxygen on it.
455 Rather than parse a complete Doxygen config file for SCons sources
456 and targets, this Doxygen builder builds a Doxygen config file,
457 adding INPUT, FILE_PATTERNS, RECURSIVE, EXCLUDE, XX_OUTPUT and
458 GENERATE_XX options (and possibly others) to an existing
459 proto-config file. Generated settings will override those in
460 the proto-config file.
462 Parameters
463 ----------
464 config : `str`
465 A Doxygen config file, usually with the extension .conf.in; a new file
466 with the ``.in`` removed will be generated and passed to Doxygen.
467 Settings in the original config file will be overridden by those
468 generated by this method.
469 **kwargs
470 Keyword arguments.
472 - ``inputs`` : A sequence of folders or files to be passed
473 as the INPUT setting for Doxygen. This list
474 will be turned into absolute paths by SCons,
475 so the ``#folder`` syntax will work.
476 Otherwise, the list is passed in as-is, but
477 the builder will also examine those
478 directories to find which source files the
479 Doxygen output actually depends on.
480 - ``patterns`` : A sequence of glob patterns for the
481 FILE_PATTERNS Doxygen setting. This will be
482 passed directly to Doxygen, but it is also
483 used to determine which source files should
484 be considered dependencies.
485 - ``recursive`` : Whether the inputs should be searched
486 recursively (used for the Doxygen RECURSIVE
487 setting).
488 - ``outputs`` : A sequence of output formats which will also
489 be used as output directories.
490 - ``exclude`` : A sequence of folders or files (not globs)
491 to be ignored by Doxygen (the Doxygen
492 EXCLUDE setting). Hidden directories are
493 automatically ignored.
494 - ``includes`` : A sequence of Doxygen config files to
495 include. These will automatically be
496 separated into paths and files to fill in
497 the ``@INCLUDE_PATH`` and ``@INCLUDE`` settings.
498 - ``useTags`` : A sequence of Doxygen tag files to use. It
499 will be assumed that the html directory for
500 each tag file is in an "html" subdirectory
501 in the same directory as the tag file.
502 - ``makeTag`` A string indicating the name of a tag file
503 to be generated.
504 - ``projectName`` : Sets the Doxygen PROJECT_NAME setting.
505 - ``projectNumber`` : Sets the Doxygen PROJECT_NUMBER setting.
506 - ``excludeSwig`` : If True (default), looks for SWIG .i files
507 in the input directories and adds Python
508 and C++ files generated by SWIG to the
509 list of files to exclude. For this to work,
510 the SWIG-generated filenames must be the
511 default ones ("module.i" generates "module.py"
512 and "moduleLib_wrap.cc").
514 Notes
515 -----
516 When building documentation from a clean source tree, generated source
517 files (like headers generated with M4) will not be included among the
518 dependencies, because they aren't present when we walk the input folders.
519 The workaround is just to build the docs after building the source.
520 """
522 inputs = [d for d in ["#doc", "#include", "#python", "#src"]
523 if os.path.exists(SCons.Script.Entry(d).abspath)]
524 defaults = {
525 "inputs": inputs,
526 "recursive": True,
527 "patterns": ["*.h", "*.cc", "*.py", "*.dox"],
528 "outputs": ["html", "xml"],
529 "excludes": [],
530 "includes": [],
531 "useTags": [],
532 "makeTag": None,
533 "projectName": None,
534 "projectNumber": None,
535 "excludeSwig": True
536 }
537 for k in defaults:
538 if kwargs.get(k) is None:
539 kwargs[k] = defaults[k]
540 builder = DoxygenBuilder(**kwargs)
541 return builder(self, config)
544@memberOf(SConsEnvironment)
545def VersionModule(self, filename, versionString=None):
546 if versionString is None:
547 for n in ("git", "hg", "svn",):
548 if os.path.isdir(".%s" % n):
549 versionString = n
551 if not versionString:
552 versionString = "git"
554 def calcMd5(filename):
555 try:
556 import hashlib
557 md5 = hashlib.md5(open(filename, "rb").read()).hexdigest()
558 except IOError:
559 md5 = None
561 return md5
563 oldMd5 = calcMd5(filename)
565 def makeVersionModule(target, source, env):
566 try:
567 version = determineVersion(state.env, versionString)
568 except RuntimeError:
569 version = "unknown"
570 parts = version.split("+")
572 names = []
573 with open(target[0].abspath, "w") as outFile:
574 outFile.write("# -------- This file is automatically generated by LSST's sconsUtils -------- #\n")
576 # Must first determine if __version_info__ is going to be
577 # included so that we can know if Tuple needs to be imported.
578 version_info = None
579 try:
580 info = tuple(int(v) for v in parts[0].split("."))
581 what = "__version_info__"
582 names.append(what)
583 version_info = f"{what} : Tuple[int, ...] = {info!r}\n"
584 except ValueError:
585 pass
587 tuple_txt = ", Tuple" if version_info is not None else ""
588 outFile.write(f"from typing import Dict, Optional{tuple_txt}\n")
589 outFile.write("\n\n")
591 what = "__version__"
592 outFile.write(f'{what}: str = "{version}"\n')
593 names.append(what)
595 what = "__repo_version__"
596 outFile.write(f'{what}: str = "{parts[0]}"\n')
597 names.append(what)
599 what = "__fingerprint__"
600 outFile.write(f'{what}: str = "{getFingerprint(versionString)}"\n')
601 names.append(what)
603 if version_info is not None:
604 outFile.write(version_info)
606 if len(parts) > 1:
607 try:
608 what = "__rebuild_version__"
609 outFile.write(f"{what}: int = {int(parts[1])}\n")
610 names.append(what)
611 except ValueError:
612 pass
614 what = "__dependency_versions__"
615 names.append(what)
616 outFile.write(f"{what}: Dict[str, Optional[str]] = {{")
617 if env.dependencies.packages:
618 outFile.write("\n")
619 for name, mod in env.dependencies.packages.items():
620 if mod is None:
621 outFile.write(f' "{name}": None,\n')
622 elif hasattr(mod.config, "version"):
623 outFile.write(f' "{name}": "{mod.config.version}",\n')
624 else:
625 outFile.write(f' "{name}": "unknown",\n')
626 outFile.write("}\n")
628 # Write out an entry per line as there can be many names
629 outFile.write("__all__ = (\n")
630 for n in names:
631 outFile.write(f' "{n}",\n')
632 outFile.write(")\n")
634 if calcMd5(target[0].abspath) != oldMd5: # only print if something's changed
635 state.log.info("makeVersionModule([\"%s\"], [])" % str(target[0]))
637 result = self.Command(filename, [], self.Action(makeVersionModule, strfunction=lambda *args: None))
639 self.AlwaysBuild(result)
640 return result