Coverage for python/lsst/ctrl/mpexec/util.py: 14%

55 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-01 10:07 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Few utility methods used by the rest of a package. 

23""" 

24 

25__all__ = ["profile", "printTable", "filterTasks", "subTaskIter"] 

26 

27# ------------------------------- 

28# Imports of standard modules -- 

29# ------------------------------- 

30import contextlib 

31import logging 

32from typing import Iterator, Optional 

33 

34# ----------------------------- 

35# Imports for other modules -- 

36# ----------------------------- 

37import lsst.pex.config as pexConfig 

38from lsst.pipe.base import Pipeline, TaskDef 

39 

40# ---------------------------------- 

41# Local non-exported definitions -- 

42# ---------------------------------- 

43 

44# ------------------------ 

45# Exported definitions -- 

46# ------------------------ 

47 

48 

49@contextlib.contextmanager 

50def profile(filename: str, log: Optional[logging.Logger] = None) -> Iterator: 

51 """Context manager for profiling with cProfile 

52 

53 Parameters 

54 ---------- 

55 filename : 

56 Filename to which to write profile (profiling disabled if `None` 

57 or empty). 

58 log : 

59 Log object for logging the profile operations. 

60 

61 Yields 

62 ------ 

63 profile : `cProfile.Profile` or `None` 

64 If profiling is enabled, the context manager returns the 

65 `cProfile.Profile` object (otherwise it returns `None`) 

66 

67 Notes 

68 ----- 

69 The returned profile object allows additional control 

70 over profiling. You can obtain this using the ``as`` clause, e.g.: 

71 

72 .. code-block:: py 

73 

74 with profile(filename) as prof: 

75 runYourCodeHere() 

76 

77 The output cumulative profile can be printed with a command-line like: 

78 

79 .. code-block:: bash 

80 

81 python -c 'import pstats; pstats.Stats("<filename>").\ 

82 sort_stats("cumtime").print_stats(30)' 

83 """ 

84 if not filename: 

85 # Nothing to do 

86 yield 

87 return 

88 from cProfile import Profile 

89 

90 prof = Profile() 

91 if log is not None: 

92 log.info("Enabling cProfile profiling") 

93 prof.enable() 

94 yield prof 

95 prof.disable() 

96 prof.dump_stats(filename) 

97 if log is not None: 

98 log.info("cProfile stats written to %s" % filename) 

99 

100 

101def printTable(rows: list[tuple], header: Optional[tuple]) -> None: 

102 """Nice formatting of 2-column table. 

103 

104 Parameters 

105 ---------- 

106 rows : `list` of `tuple` 

107 Each item in the list is a 2-tuple containg left and righ column values 

108 header: `tuple` or `None` 

109 If `None` then table header are not prined, otherwise it's a 2-tuple 

110 with column headings. 

111 """ 

112 if not rows: 

113 return 

114 width = max(len(x[0]) for x in rows) 

115 if header: 

116 width = max(width, len(header[0])) 

117 print(header[0].ljust(width), header[1]) 

118 print("".ljust(width, "-"), "".ljust(len(header[1]), "-")) 

119 for col1, col2 in rows: 

120 print(col1.ljust(width), col2) 

121 

122 

123def filterTasks(pipeline: Pipeline, name: Optional[str]) -> list[TaskDef]: 

124 """Finds list of tasks matching given name. 

125 

126 For matching task either task label or task name after last dot should 

127 be identical to `name`. If task label is non-empty then task name is not 

128 checked. 

129 

130 Parameters 

131 ---------- 

132 pipeline : `Pipeline` 

133 name : str or none 

134 If empty or None then all tasks are returned 

135 

136 Returns 

137 ------- 

138 Lsit of `TaskDef` instances. 

139 """ 

140 if not name: 

141 return list(pipeline.toExpandedPipeline()) 

142 tasks = [] 

143 for taskDef in pipeline.toExpandedPipeline(): 

144 if taskDef.label: 

145 if taskDef.label == name: 

146 tasks.append(taskDef) 

147 elif taskDef.taskName.split(".")[-1] == name: 

148 tasks.append(taskDef) 

149 return tasks 

150 

151 

152def subTaskIter(config: pexConfig.Config) -> Iterator[tuple[str, str]]: 

153 """Recursively generates subtask names. 

154 

155 Parameters 

156 ---------- 

157 config : `lsst.pex.config.Config` 

158 Configuration of the task 

159 

160 Returns 

161 ------- 

162 Iterator which returns tuples of (configFieldPath, taskName). 

163 """ 

164 for fieldName, field in sorted(config.items()): 

165 if hasattr(field, "value") and hasattr(field, "target"): 

166 subConfig = field.value 

167 if isinstance(subConfig, pexConfig.Config): 

168 try: 

169 taskName = "%s.%s" % (field.target.__module__, field.target.__name__) 

170 except Exception: 

171 taskName = repr(field.target) 

172 yield fieldName, taskName 

173 for subFieldName, taskName in subTaskIter(subConfig): 

174 yield fieldName + "." + subFieldName, taskName