Coverage for python/lsst/ctrl/bps/wms_service.py: 82%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

69 statements  

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Base classes for working with a specific WMS""" 

23 

24 

25__all__ = ["BaseWmsService", "BaseWmsWorkflow", "WmsJobReport", "WmsRunReport", "WmsStates"] 

26 

27 

28import logging 

29import dataclasses 

30from abc import ABCMeta 

31from enum import Enum 

32 

33 

34_LOG = logging.getLogger(__name__) 

35 

36 

37class WmsStates(Enum): 

38 """Run and job states 

39 """ 

40 UNKNOWN = 0 # Can't determine state 

41 MISFIT = 1 # Determined state, but doesn't fit other states 

42 UNREADY = 2 # Still waiting for parents to finish 

43 READY = 3 # All of its parents have finished successfully 

44 PENDING = 4 # Ready to run, visible in batch queue 

45 RUNNING = 5 # Currently running 

46 DELETED = 6 # In the process of being deleted or already deleted 

47 HELD = 7 # In a hold state 

48 SUCCEEDED = 8 # Have completed with success status 

49 FAILED = 9 # Have completed with non-success status 

50 

51 

52@dataclasses.dataclass 

53class WmsJobReport: 

54 """WMS job information to be included in detailed report output 

55 """ 

56 wms_id: str 

57 name: str 

58 label: str 

59 state: WmsStates 

60 

61 __slots__ = ('wms_id', 'name', 'label', 'state') 

62 

63 

64@dataclasses.dataclass 

65class WmsRunReport: 

66 """WMS run information to be included in detailed report output 

67 """ 

68 wms_id: str 

69 global_wms_id: str 

70 path: str 

71 label: str 

72 run: str 

73 project: str 

74 campaign: str 

75 payload: str 

76 operator: str 

77 run_summary: str 

78 state: WmsStates 

79 jobs: list 

80 total_number_jobs: int 

81 job_state_counts: dict 

82 

83 __slots__ = ('wms_id', 'global_wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload', 

84 'operator', 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts') 

85 

86 

87class BaseWmsService: 

88 """Interface for interactions with a specific WMS. 

89 

90 Parameters 

91 ---------- 

92 config : `lsst.ctrl.bps.BpsConfig` 

93 Configuration needed by the WMS service. 

94 """ 

95 def __init__(self, config): 

96 self.config = config 

97 

98 def prepare(self, config, generic_workflow, out_prefix=None): 

99 """Create submission for a generic workflow for a specific WMS. 

100 

101 Parameters 

102 ---------- 

103 config : `lsst.ctrl.bps.BpsConfig` 

104 BPS configuration. 

105 generic_workflow : `lsst.ctrl.bps.GenericWorkflow` 

106 Generic representation of a single workflow 

107 out_prefix : `str` 

108 Prefix for all WMS output files 

109 

110 Returns 

111 ------- 

112 wms_workflow : `BaseWmsWorkflow` 

113 Prepared WMS Workflow to submit for execution 

114 """ 

115 raise NotImplementedError 

116 

117 def submit(self, workflow): 

118 """Submit a single WMS workflow 

119 

120 Parameters 

121 ---------- 

122 workflow : `lsst.ctrl.bps.BaseWmsWorkflow` 

123 Prepared WMS Workflow to submit for execution 

124 """ 

125 raise NotImplementedError 

126 

127 def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thru=None, is_global=False): 

128 """Query WMS for list of submitted WMS workflows/jobs. 

129 

130 This should be a quick lookup function to create list of jobs for 

131 other functions. 

132 

133 Parameters 

134 ---------- 

135 wms_id : `int` or `str`, optional 

136 Id or path that can be used by WMS service to look up job. 

137 user : `str`, optional 

138 User whose submitted jobs should be listed. 

139 require_bps : `bool`, optional 

140 Whether to require jobs returned in list to be bps-submitted jobs. 

141 pass_thru : `str`, optional 

142 Information to pass through to WMS. 

143 is_global : `bool`, optional 

144 If set, all available job queues will be queried for job 

145 information. Defaults to False which means that only a local job 

146 queue will be queried for information. 

147 

148 Only applicable in the context of a WMS using distributed job 

149 queues (e.g., HTCondor). A WMS with a centralized job queue 

150 (e.g. PanDA) can safely ignore it. 

151 

152 Returns 

153 ------- 

154 job_ids : `list` [`Any`] 

155 Only job ids to be used by cancel and other functions. Typically 

156 this means top-level jobs (i.e., not children jobs). 

157 """ 

158 raise NotImplementedError 

159 

160 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None, is_global=False): 

161 """Query WMS for status of submitted WMS workflows. 

162 

163 Parameters 

164 ---------- 

165 wms_workflow_id : `int` or `str`, optional 

166 Id that can be used by WMS service to look up status. 

167 user : `str`, optional 

168 Limit report to submissions by this particular user. 

169 hist : `int`, optional 

170 Number of days to expand report to include finished WMS workflows. 

171 pass_thru : `str`, optional 

172 Additional arguments to pass through to the specific WMS service. 

173 is_global : `bool`, optional 

174 If set, all available job queues will be queried for job 

175 information. Defaults to False which means that only a local job 

176 queue will be queried for information. 

177 

178 Only applicable in the context of a WMS using distributed job 

179 queues (e.g., HTCondor). A WMS with a centralized job queue 

180 (e.g. PanDA) can safely ignore it. 

181 

182 Returns 

183 ------- 

184 run_reports : `list` [`lsst.ctrl.bps.WmsRunReport`] 

185 Status information for submitted WMS workflows. 

186 message : `str` 

187 Message to user on how to find more status information specific to 

188 this particular WMS. 

189 """ 

190 raise NotImplementedError 

191 

192 def cancel(self, wms_id, pass_thru=None): 

193 """Cancel submitted workflows/jobs. 

194 

195 Parameters 

196 ---------- 

197 wms_id : `str` 

198 ID or path of job that should be canceled. 

199 pass_thru : `str`, optional 

200 Information to pass through to WMS. 

201 

202 Returns 

203 -------- 

204 deleted : `bool` 

205 Whether successful deletion or not. Currently, if any doubt or any 

206 individual jobs not deleted, return False. 

207 message : `str` 

208 Any message from WMS (e.g., error details). 

209 """ 

210 raise NotImplementedError 

211 

212 def run_submission_checks(self): 

213 """Checks to run at start if running WMS specific submission steps. 

214 

215 Any exception other than NotImplementedError will halt submission. 

216 Submit directory may not yet exist when this is called. 

217 """ 

218 raise NotImplementedError 

219 

220 

221class BaseWmsWorkflow(metaclass=ABCMeta): 

222 """Interface for single workflow specific to a WMS. 

223 

224 Parameters 

225 ---------- 

226 name : `str` 

227 Unique name of workflow. 

228 config : `lsst.ctrl.bps.BpsConfig` 

229 Generic workflow config. 

230 """ 

231 def __init__(self, name, config): 

232 self.name = name 

233 self.config = config 

234 self.service_class = None 

235 self.run_id = None 

236 self.submit_path = None 

237 

238 @classmethod 

239 def from_generic_workflow(cls, config, generic_workflow, out_prefix, 

240 service_class): 

241 """Create a WMS-specific workflow from a GenericWorkflow 

242 

243 Parameters 

244 ---------- 

245 config : `lsst.ctrl.bps.BpsConfig` 

246 Configuration values needed for generating a WMS specific workflow. 

247 generic_workflow : `lsst.ctrl.bps.GenericWorkflow` 

248 Generic workflow from which to create the WMS-specific one. 

249 out_prefix : `str` 

250 Root directory to be used for WMS workflow inputs and outputs 

251 as well as internal WMS files. 

252 service_class : `str` 

253 Full module name of WMS service class that created this workflow. 

254 

255 Returns 

256 ------- 

257 wms_workflow : `lsst.ctrl.bps.BaseWmsWorkflow` 

258 A WMS specific workflow. 

259 """ 

260 

261 raise NotImplementedError 

262 

263 def write(self, out_prefix): 

264 """Write WMS files for this particular workflow. 

265 

266 Parameters 

267 ---------- 

268 out_prefix : `str` 

269 Root directory to be used for WMS workflow inputs and outputs 

270 as well as internal WMS files. 

271 """ 

272 raise NotImplementedError