Coverage for python/lsst/ctrl/mpexec/util.py: 16%

55 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-09 12:06 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Few utility methods used by the rest of a package. 

29""" 

30 

31__all__ = ["profile", "printTable", "filterTasks", "subTaskIter"] 

32 

33# ------------------------------- 

34# Imports of standard modules -- 

35# ------------------------------- 

36import contextlib 

37import logging 

38from collections.abc import Iterator 

39 

40# ----------------------------- 

41# Imports for other modules -- 

42# ----------------------------- 

43import lsst.pex.config as pexConfig 

44from lsst.pipe.base import Pipeline, TaskDef 

45 

46# ---------------------------------- 

47# Local non-exported definitions -- 

48# ---------------------------------- 

49 

50# ------------------------ 

51# Exported definitions -- 

52# ------------------------ 

53 

54 

55@contextlib.contextmanager 

56def profile(filename: str, log: logging.Logger | None = None) -> Iterator: 

57 """Context manager for profiling with cProfile 

58 

59 Parameters 

60 ---------- 

61 filename : 

62 Filename to which to write profile (profiling disabled if `None` 

63 or empty). 

64 log : 

65 Log object for logging the profile operations. 

66 

67 Yields 

68 ------ 

69 profile : `cProfile.Profile` or `None` 

70 If profiling is enabled, the context manager returns the 

71 `cProfile.Profile` object (otherwise it returns `None`) 

72 

73 Notes 

74 ----- 

75 The returned profile object allows additional control 

76 over profiling. You can obtain this using the ``as`` clause, e.g.: 

77 

78 .. code-block:: py 

79 

80 with profile(filename) as prof: 

81 runYourCodeHere() 

82 

83 The output cumulative profile can be printed with a command-line like: 

84 

85 .. code-block:: bash 

86 

87 python -c 'import pstats; pstats.Stats("<filename>").\ 

88 sort_stats("cumtime").print_stats(30)' 

89 """ 

90 if not filename: 

91 # Nothing to do 

92 yield 

93 return 

94 from cProfile import Profile 

95 

96 prof = Profile() 

97 if log is not None: 

98 log.info("Enabling cProfile profiling") 

99 prof.enable() 

100 yield prof 

101 prof.disable() 

102 prof.dump_stats(filename) 

103 if log is not None: 

104 log.info("cProfile stats written to %s" % filename) 

105 

106 

107def printTable(rows: list[tuple], header: tuple | None) -> None: 

108 """Nice formatting of 2-column table. 

109 

110 Parameters 

111 ---------- 

112 rows : `list` of `tuple` 

113 Each item in the list is a 2-tuple containg left and righ column values 

114 header: `tuple` or `None` 

115 If `None` then table header are not prined, otherwise it's a 2-tuple 

116 with column headings. 

117 """ 

118 if not rows: 

119 return 

120 width = max(len(x[0]) for x in rows) 

121 if header: 

122 width = max(width, len(header[0])) 

123 print(header[0].ljust(width), header[1]) 

124 print("".ljust(width, "-"), "".ljust(len(header[1]), "-")) 

125 for col1, col2 in rows: 

126 print(col1.ljust(width), col2) 

127 

128 

129def filterTasks(pipeline: Pipeline, name: str | None) -> list[TaskDef]: 

130 """Find list of tasks matching given name. 

131 

132 For matching task either task label or task name after last dot should 

133 be identical to `name`. If task label is non-empty then task name is not 

134 checked. 

135 

136 Parameters 

137 ---------- 

138 pipeline : `Pipeline` 

139 Pipeline to examine. 

140 name : `str` or `None` 

141 If empty or `None` then all tasks are returned. 

142 

143 Returns 

144 ------- 

145 tasks : `list` [ `lsst.pipe.base.TaskDef`] 

146 List of `~lsst.pipe.base.TaskDef` instances that match. 

147 """ 

148 if not name: 

149 return list(pipeline.toExpandedPipeline()) 

150 tasks = [] 

151 for taskDef in pipeline.toExpandedPipeline(): 

152 if taskDef.label: 

153 if taskDef.label == name: 

154 tasks.append(taskDef) 

155 elif taskDef.taskName.split(".")[-1] == name: 

156 tasks.append(taskDef) 

157 return tasks 

158 

159 

160def subTaskIter(config: pexConfig.Config) -> Iterator[tuple[str, str]]: 

161 """Recursively generates subtask names. 

162 

163 Parameters 

164 ---------- 

165 config : `lsst.pex.config.Config` 

166 Configuration of the task 

167 

168 Returns 

169 ------- 

170 names : `collections.abc.Iterator` [ `tuple` [ `str`, `str` ] ] 

171 Iterator which returns tuples of (configFieldPath, taskName). 

172 """ 

173 for fieldName, field in sorted(config.items()): 

174 if hasattr(field, "value") and hasattr(field, "target"): 

175 subConfig = field.value 

176 if isinstance(subConfig, pexConfig.Config): 

177 try: 

178 taskName = f"{field.target.__module__}.{field.target.__name__}" 

179 except Exception: 

180 taskName = repr(field.target) 

181 yield fieldName, taskName 

182 for subFieldName, taskName in subTaskIter(subConfig): 

183 yield fieldName + "." + subFieldName, taskName