Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Supporting functions for reporting on runs submitted to a WMS 

23 

24Note: Expectations are that future reporting effort will revolve around 

25 LSST oriented database tables. 

26""" 

27 

28import logging 

29 

30from lsst.utils import doImport 

31 

32from .wms_service import WmsStates 

33 

34 

35SUMMARY_FMT = "{:1} {:>10} {:>3} {:>9} {:10} {:10} {:20} {:20} {:<60}" 

36 

37 

38_LOG = logging.getLogger(__name__) 

39 

40 

41def report(wms_service, run_id, user, hist_days, pass_thru): 

42 """Print out summary of jobs submitted for execution. 

43 

44 Parameters 

45 ---------- 

46 wms_service : `str` 

47 Name of the class. 

48 run_id : `str` 

49 A run id the report will be restricted to. 

50 user : `str` 

51 A user name the report will be restricted to. 

52 hist_days : int 

53 Number of days 

54 pass_thru : `str` 

55 A string to pass directly to the WMS service class. 

56 """ 

57 wms_service_class = doImport(wms_service) 

58 wms_service = wms_service_class({}) 

59 

60 # If reporting on single run, increase history until better mechanism 

61 # for handling completed jobs is available. 

62 if run_id: 

63 hist_days = max(hist_days, 2) 

64 

65 runs, message = wms_service.report(run_id, user, hist_days, pass_thru) 

66 

67 if run_id: 

68 if not runs: 

69 print(f"No information found for id='{run_id}'.") 

70 print(f"Double check id and retry with a larger --hist value" 

71 f"(currently: {hist_days})") 

72 for run in runs: 

73 print_single_run_summary(run) 

74 else: 

75 print_headers() 

76 for run in sorted(runs, key=lambda j: j.wms_id): 

77 print_run(run) 

78 print(message) 

79 

80 

81def print_headers(): 

82 """Print headers. 

83 """ 

84 print(SUMMARY_FMT.format("X", "STATE", "%S", "ID", "OPERATOR", "PRJ", "CMPGN", "PAYLOAD", "RUN")) 

85 print("-" * 156) 

86 

87 

88def print_run(run_report): 

89 """Print single run info. 

90 

91 Parameters 

92 ---------- 

93 run_report : `WmsRunReport` 

94 Information for single run. 

95 """ 

96 # Flag any running workflow that might need human attention 

97 run_flag = " " 

98 if run_report.state == WmsStates.RUNNING: 

99 if run_report.job_state_counts.get(WmsStates.FAILED, 0): 

100 run_flag = "F" 

101 elif run_report.job_state_counts.get(WmsStates.DELETED, 0): 

102 run_flag = "D" 

103 elif run_report.job_state_counts.get(WmsStates.HELD, 0): 

104 run_flag = "H" 

105 

106 percent_succeeded = "UNK" 

107 _LOG.debug("total_number_jobs = %s", run_report.total_number_jobs) 

108 _LOG.debug("run_report.job_state_counts = %s", run_report.job_state_counts) 

109 if run_report.total_number_jobs: 

110 succeeded = run_report.job_state_counts.get(WmsStates.SUCCEEDED, 0) 

111 _LOG.debug("succeeded = %s", succeeded) 

112 percent_succeeded = f"{int(succeeded / run_report.total_number_jobs * 100)}" 

113 

114 print(SUMMARY_FMT.format(run_flag, run_report.state.name, percent_succeeded, run_report.wms_id, 

115 run_report.operator[:10], run_report.project[:10], run_report.campaign[:20], 

116 run_report.payload[:20], run_report.run[:60])) 

117 

118 

119def group_jobs_by_state(jobs): 

120 """Divide given jobs into groups based on their state value. 

121 

122 Parameters 

123 ---------- 

124 jobs : `list` of `~lsst.ctrl.bps.wms_service.WmsJobReport` 

125 Jobs to divide into groups based on state. 

126 

127 Returns 

128 ------- 

129 by_state : `dict` 

130 Mapping of job state to a list of jobs. 

131 """ 

132 _LOG.debug("group_jobs_by_state: jobs=%s", jobs) 

133 by_state = dict.fromkeys(WmsStates) 

134 for state in by_state: 

135 by_state[state] = [] # Note: If added [] to fromkeys(), they shared single list. 

136 

137 for job in jobs: 

138 by_state[job.state].append(job) 

139 return by_state 

140 

141 

142def group_jobs_by_label(jobs): 

143 """Divide given jobs into groups based on their label value. 

144 

145 Parameters 

146 ---------- 

147 jobs : `list` of `~lsst.ctrl.bps.wms_service.WmsJobReport` 

148 Jobs to divide into groups based on label. 

149 

150 Returns 

151 ------- 

152 by_label : `dict` 

153 Mapping of job state to a list of jobs . 

154 """ 

155 by_label = {} 

156 for job in jobs: 

157 if job.label not in by_label: 

158 by_label[job.label] = [] 

159 by_label[job.label].append(job) 

160 return by_label 

161 

162 

163def print_single_run_summary(run_report): 

164 """Print runtime info for single run including job summary per task abbrev. 

165 

166 Parameters 

167 ---------- 

168 run_report : `~lsst.ctrl.bps.wms_service.WmsRunReport` 

169 Summary runtime info for a run + runtime info for jobs. 

170 """ 

171 # Print normal run summary. 

172 print_headers() 

173 print_run(run_report) 

174 print("\n\n") 

175 

176 # Print more run information. 

177 print(f"Path: {run_report.path}\n") 

178 

179 print(f"{'':35} {' | '.join([f'{s.name[:6]:6}' for s in WmsStates])}") 

180 print(f"{'Total':35} {' | '.join([f'{run_report.job_state_counts[s]:6}' for s in WmsStates])}") 

181 print("-" * (35 + 3 + (6 + 2) * (len(run_report.job_state_counts) + 1))) 

182 

183 by_label = group_jobs_by_label(run_report.jobs) 

184 

185 # Print job level info by print counts of jobs by label and WMS state. 

186 label_order = [] 

187 by_label_totals = {} 

188 if run_report.run_summary: 

189 # Workaround until get pipetaskInit job into run_summary 

190 if not run_report.run_summary.startswith("pipetaskInit"): 

191 label_order.append("pipetaskInit") 

192 by_label_totals["pipetaskInit"] = 1 

193 for part in run_report.run_summary.split(";"): 

194 label, count = part.split(":") 

195 label_order.append(label) 

196 by_label_totals[label] = int(count) 

197 else: 

198 print("Warning: Cannot determine order of pipeline. Instead printing alphabetical.") 

199 label_order = sorted(by_label.keys()) 

200 

201 for label in label_order: 

202 counts = dict.fromkeys(WmsStates, 0) 

203 if label in by_label: 

204 by_label_state = group_jobs_by_state(by_label[label]) 

205 _LOG.debug("by_label_state = %s", by_label_state) 

206 counts = dict.fromkeys(WmsStates) 

207 for state in WmsStates: 

208 counts[state] = len(by_label_state[state]) 

209 elif label in by_label_totals: 

210 already_counted = sum(counts.values()) 

211 if already_counted != by_label_totals[label]: 

212 counts[WmsStates.UNREADY] += by_label_totals[label] - already_counted 

213 else: 

214 counts = dict.fromkeys(WmsStates, -1) 

215 print(f"{label[:35]:35} {' | '.join([f'{counts[s]:6}' for s in WmsStates])}") 

216 print("\n")