Coverage for python/lsst/ctrl/mpexec/util.py: 14%

55 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-11 09:04 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Few utility methods used by the rest of a package. 

23""" 

24 

25__all__ = ["profile", "printTable", "filterTasks", "subTaskIter"] 

26 

27# ------------------------------- 

28# Imports of standard modules -- 

29# ------------------------------- 

30import contextlib 

31import logging 

32from collections.abc import Iterator 

33 

34# ----------------------------- 

35# Imports for other modules -- 

36# ----------------------------- 

37import lsst.pex.config as pexConfig 

38from lsst.pipe.base import Pipeline, TaskDef 

39 

40# ---------------------------------- 

41# Local non-exported definitions -- 

42# ---------------------------------- 

43 

44# ------------------------ 

45# Exported definitions -- 

46# ------------------------ 

47 

48 

49@contextlib.contextmanager 

50def profile(filename: str, log: logging.Logger | None = None) -> Iterator: 

51 """Context manager for profiling with cProfile 

52 

53 Parameters 

54 ---------- 

55 filename : 

56 Filename to which to write profile (profiling disabled if `None` 

57 or empty). 

58 log : 

59 Log object for logging the profile operations. 

60 

61 Yields 

62 ------ 

63 profile : `cProfile.Profile` or `None` 

64 If profiling is enabled, the context manager returns the 

65 `cProfile.Profile` object (otherwise it returns `None`) 

66 

67 Notes 

68 ----- 

69 The returned profile object allows additional control 

70 over profiling. You can obtain this using the ``as`` clause, e.g.: 

71 

72 .. code-block:: py 

73 

74 with profile(filename) as prof: 

75 runYourCodeHere() 

76 

77 The output cumulative profile can be printed with a command-line like: 

78 

79 .. code-block:: bash 

80 

81 python -c 'import pstats; pstats.Stats("<filename>").\ 

82 sort_stats("cumtime").print_stats(30)' 

83 """ 

84 if not filename: 

85 # Nothing to do 

86 yield 

87 return 

88 from cProfile import Profile 

89 

90 prof = Profile() 

91 if log is not None: 

92 log.info("Enabling cProfile profiling") 

93 prof.enable() 

94 yield prof 

95 prof.disable() 

96 prof.dump_stats(filename) 

97 if log is not None: 

98 log.info("cProfile stats written to %s" % filename) 

99 

100 

101def printTable(rows: list[tuple], header: tuple | None) -> None: 

102 """Nice formatting of 2-column table. 

103 

104 Parameters 

105 ---------- 

106 rows : `list` of `tuple` 

107 Each item in the list is a 2-tuple containg left and righ column values 

108 header: `tuple` or `None` 

109 If `None` then table header are not prined, otherwise it's a 2-tuple 

110 with column headings. 

111 """ 

112 if not rows: 

113 return 

114 width = max(len(x[0]) for x in rows) 

115 if header: 

116 width = max(width, len(header[0])) 

117 print(header[0].ljust(width), header[1]) 

118 print("".ljust(width, "-"), "".ljust(len(header[1]), "-")) 

119 for col1, col2 in rows: 

120 print(col1.ljust(width), col2) 

121 

122 

123def filterTasks(pipeline: Pipeline, name: str | None) -> list[TaskDef]: 

124 """Find list of tasks matching given name. 

125 

126 For matching task either task label or task name after last dot should 

127 be identical to `name`. If task label is non-empty then task name is not 

128 checked. 

129 

130 Parameters 

131 ---------- 

132 pipeline : `Pipeline` 

133 Pipeline to examine. 

134 name : `str` or `None` 

135 If empty or `None` then all tasks are returned. 

136 

137 Returns 

138 ------- 

139 tasks : `list` [ `lsst.pipe.base.TaskDef`] 

140 List of `~lsst.pipe.base.TaskDef` instances that match. 

141 """ 

142 if not name: 

143 return list(pipeline.toExpandedPipeline()) 

144 tasks = [] 

145 for taskDef in pipeline.toExpandedPipeline(): 

146 if taskDef.label: 

147 if taskDef.label == name: 

148 tasks.append(taskDef) 

149 elif taskDef.taskName.split(".")[-1] == name: 

150 tasks.append(taskDef) 

151 return tasks 

152 

153 

154def subTaskIter(config: pexConfig.Config) -> Iterator[tuple[str, str]]: 

155 """Recursively generates subtask names. 

156 

157 Parameters 

158 ---------- 

159 config : `lsst.pex.config.Config` 

160 Configuration of the task 

161 

162 Returns 

163 ------- 

164 names : `collections.abc.Iterator` [ `tuple` [ `str`, `str` ] ] 

165 Iterator which returns tuples of (configFieldPath, taskName). 

166 """ 

167 for fieldName, field in sorted(config.items()): 

168 if hasattr(field, "value") and hasattr(field, "target"): 

169 subConfig = field.value 

170 if isinstance(subConfig, pexConfig.Config): 

171 try: 

172 taskName = "%s.%s" % (field.target.__module__, field.target.__name__) 

173 except Exception: 

174 taskName = repr(field.target) 

175 yield fieldName, taskName 

176 for subFieldName, taskName in subTaskIter(subConfig): 

177 yield fieldName + "." + subFieldName, taskName