Coverage for python/lsst/ctrl/bps/wms_service.py: 82%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

68 statements  

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Base classes for working with a specific WMS""" 

23 

24 

25__all__ = ["BaseWmsService", "BaseWmsWorkflow", "WmsJobReport", "WmsRunReport", "WmsStates"] 

26 

27 

28import logging 

29import dataclasses 

30from abc import ABCMeta 

31from enum import Enum 

32 

33 

34_LOG = logging.getLogger(__name__) 

35 

36 

37class WmsStates(Enum): 

38 """Run and job states 

39 """ 

40 UNKNOWN = 0 # Can't determine state 

41 MISFIT = 1 # Determined state, but doesn't fit other states 

42 UNREADY = 2 # Still waiting for parents to finish 

43 READY = 3 # All of its parents have finished successfully 

44 PENDING = 4 # Ready to run, visible in batch queue 

45 RUNNING = 5 # Currently running 

46 DELETED = 6 # In the process of being deleted or already deleted 

47 HELD = 7 # In a hold state 

48 SUCCEEDED = 8 # Have completed with success status 

49 FAILED = 9 # Have completed with non-success status 

50 

51 

52@dataclasses.dataclass 

53class WmsJobReport: 

54 """WMS job information to be included in detailed report output 

55 """ 

56 wms_id: str 

57 name: str 

58 label: str 

59 state: WmsStates 

60 

61 __slots__ = ('wms_id', 'name', 'label', 'state') 

62 

63 

64@dataclasses.dataclass 

65class WmsRunReport: 

66 """WMS run information to be included in detailed report output 

67 """ 

68 wms_id: str 

69 path: str 

70 label: str 

71 run: str 

72 project: str 

73 campaign: str 

74 payload: str 

75 operator: str 

76 run_summary: str 

77 state: WmsStates 

78 jobs: list 

79 total_number_jobs: int 

80 job_state_counts: dict 

81 

82 __slots__ = ('wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload', 'operator', 

83 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts') 

84 

85 

86class BaseWmsService: 

87 """Interface for interactions with a specific WMS. 

88 

89 Parameters 

90 ---------- 

91 config : `lsst.ctrl.bps.BpsConfig` 

92 Configuration needed by the WMS service. 

93 """ 

94 def __init__(self, config): 

95 self.config = config 

96 

97 def prepare(self, config, generic_workflow, out_prefix=None): 

98 """Create submission for a generic workflow for a specific WMS. 

99 

100 Parameters 

101 ---------- 

102 config : `lsst.ctrl.bps.BpsConfig` 

103 BPS configuration. 

104 generic_workflow : `lsst.ctrl.bps.GenericWorkflow` 

105 Generic representation of a single workflow 

106 out_prefix : `str` 

107 Prefix for all WMS output files 

108 

109 Returns 

110 ------- 

111 wms_workflow : `BaseWmsWorkflow` 

112 Prepared WMS Workflow to submit for execution 

113 """ 

114 raise NotImplementedError 

115 

116 def submit(self, workflow): 

117 """Submit a single WMS workflow 

118 

119 Parameters 

120 ---------- 

121 workflow : `lsst.ctrl.bps.BaseWmsWorkflow` 

122 Prepared WMS Workflow to submit for execution 

123 """ 

124 raise NotImplementedError 

125 

126 def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thru=None): 

127 """Query WMS for list of submitted WMS workflows/jobs. 

128 

129 This should be a quick lookup function to create list of jobs for 

130 other functions. 

131 

132 Parameters 

133 ---------- 

134 wms_id : `int` or `str`, optional 

135 Id or path that can be used by WMS service to look up job. 

136 user : `str`, optional 

137 User whose submitted jobs should be listed. 

138 require_bps : `bool`, optional 

139 Whether to require jobs returned in list to be bps-submitted jobs. 

140 pass_thru : `str`, optional 

141 Information to pass through to WMS. 

142 

143 Returns 

144 ------- 

145 job_ids : `list` [`Any`] 

146 Only job ids to be used by cancel and other functions. Typically 

147 this means top-level jobs (i.e., not children jobs). 

148 """ 

149 raise NotImplementedError 

150 

151 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None): 

152 """Query WMS for status of submitted WMS workflows. 

153 

154 Parameters 

155 ---------- 

156 wms_workflow_id : `int` or `str`, optional 

157 Id that can be used by WMS service to look up status. 

158 user : `str`, optional 

159 Limit report to submissions by this particular user. 

160 hist : `int`, optional 

161 Number of days to expand report to include finished WMS workflows. 

162 pass_thru : `str`, optional 

163 Additional arguments to pass through to the specific WMS service. 

164 

165 Returns 

166 ------- 

167 run_reports : `list` [`lsst.ctrl.bps.WmsRunReport`] 

168 Status information for submitted WMS workflows. 

169 message : `str` 

170 Message to user on how to find more status information specific to 

171 this particular WMS. 

172 """ 

173 raise NotImplementedError 

174 

175 def cancel(self, wms_id, pass_thru=None): 

176 """Cancel submitted workflows/jobs. 

177 

178 Parameters 

179 ---------- 

180 wms_id : `str` 

181 ID or path of job that should be canceled. 

182 pass_thru : `str`, optional 

183 Information to pass through to WMS. 

184 

185 Returns 

186 -------- 

187 deleted : `bool` 

188 Whether successful deletion or not. Currently, if any doubt or any 

189 individual jobs not deleted, return False. 

190 message : `str` 

191 Any message from WMS (e.g., error details). 

192 """ 

193 raise NotImplementedError 

194 

195 def run_submission_checks(self): 

196 """Checks to run at start if running WMS specific submission steps. 

197 

198 Any exception other than NotImplementedError will halt submission. 

199 Submit directory may not yet exist when this is called. 

200 """ 

201 raise NotImplementedError 

202 

203 

204class BaseWmsWorkflow(metaclass=ABCMeta): 

205 """Interface for single workflow specific to a WMS. 

206 

207 Parameters 

208 ---------- 

209 name : `str` 

210 Unique name of workflow. 

211 config : `lsst.ctrl.bps.BpsConfig` 

212 Generic workflow config. 

213 """ 

214 def __init__(self, name, config): 

215 self.name = name 

216 self.config = config 

217 self.service_class = None 

218 self.run_id = None 

219 self.submit_path = None 

220 

221 @classmethod 

222 def from_generic_workflow(cls, config, generic_workflow, out_prefix, 

223 service_class): 

224 """Create a WMS-specific workflow from a GenericWorkflow 

225 

226 Parameters 

227 ---------- 

228 config : `lsst.ctrl.bps.BpsConfig` 

229 Configuration values needed for generating a WMS specific workflow. 

230 generic_workflow : `lsst.ctrl.bps.GenericWorkflow` 

231 Generic workflow from which to create the WMS-specific one. 

232 out_prefix : `str` 

233 Root directory to be used for WMS workflow inputs and outputs 

234 as well as internal WMS files. 

235 service_class : `str` 

236 Full module name of WMS service class that created this workflow. 

237 

238 Returns 

239 ------- 

240 wms_workflow : `lsst.ctrl.bps.BaseWmsWorkflow` 

241 A WMS specific workflow. 

242 """ 

243 

244 raise NotImplementedError 

245 

246 def write(self, out_prefix): 

247 """Write WMS files for this particular workflow. 

248 

249 Parameters 

250 ---------- 

251 out_prefix : `str` 

252 Root directory to be used for WMS workflow inputs and outputs 

253 as well as internal WMS files. 

254 """ 

255 raise NotImplementedError