Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Base classes for working with a specific WMS""" 

23 

24 

25__all__ = ["BaseWmsService", "BaseWmsWorkflow", "WmsJobReport", "WmsRunReport", "WmsStates"] 

26 

27 

28import logging 

29import dataclasses 

30from abc import ABCMeta 

31from enum import Enum 

32 

33 

34_LOG = logging.getLogger(__name__) 

35 

36 

37class WmsStates(Enum): 

38 """Run and job states 

39 """ 

40 UNKNOWN = 0 # Can't determine state 

41 MISFIT = 1 # Determined state, but doesn't fit other states 

42 UNREADY = 2 # Still waiting for parents to finish 

43 READY = 3 # All of its parents have finished successfully 

44 PENDING = 4 # Ready to run, visible in batch queue 

45 RUNNING = 5 # Currently running 

46 DELETED = 6 # In the process of being deleted or already deleted 

47 HELD = 7 # In a hold state 

48 SUCCEEDED = 8 # Have completed with success status 

49 FAILED = 9 # Have completed with non-success status 

50 

51 

52@dataclasses.dataclass 

53class WmsJobReport: 

54 """WMS job information to be included in detailed report output 

55 """ 

56 wms_id: str 

57 name: str 

58 label: str 

59 state: WmsStates 

60 

61 __slots__ = ('wms_id', 'name', 'label', 'state') 

62 

63 

64@dataclasses.dataclass 

65class WmsRunReport: 

66 """WMS run information to be included in detailed report output 

67 """ 

68 wms_id: str 

69 path: str 

70 label: str 

71 run: str 

72 project: str 

73 campaign: str 

74 payload: str 

75 operator: str 

76 run_summary: str 

77 state: WmsStates 

78 jobs: list 

79 total_number_jobs: int 

80 job_state_counts: dict 

81 

82 __slots__ = ('wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload', 'operator', 

83 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts') 

84 

85 

86class BaseWmsService: 

87 """Interface for interactions with a specific WMS. 

88 

89 Parameters 

90 ---------- 

91 config : `lsst.ctrl.bps.BpsConfig` 

92 Configuration needed by the WMS service. 

93 """ 

94 def __init__(self, config): 

95 self.config = config 

96 

97 def prepare(self, config, generic_workflow, out_prefix=None): 

98 """Create submission for a generic workflow for a specific WMS. 

99 

100 Parameters 

101 ---------- 

102 config : `lsst.ctrl.bps.BpsConfig` 

103 BPS configuration. 

104 generic_workflow : `lsst.ctrl.bps.GenericWorkflow` 

105 Generic representation of a single workflow 

106 out_prefix : `str` 

107 Prefix for all WMS output files 

108 

109 Returns 

110 ------- 

111 wms_workflow : `BaseWmsWorkflow` 

112 Prepared WMS Workflow to submit for execution 

113 """ 

114 raise NotImplementedError 

115 

116 def submit(self, workflow): 

117 """Submit a single WMS workflow 

118 

119 Parameters 

120 ---------- 

121 workflow : `lsst.ctrl.bps.BaseWmsWorkflow` 

122 Prepared WMS Workflow to submit for execution 

123 """ 

124 raise NotImplementedError 

125 

126 def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thru=None): 

127 """Query WMS for list of submitted WMS workflows/jobs. 

128 

129 This should be a quick lookup function to create list of jobs for 

130 other functions. 

131 

132 Parameters 

133 ---------- 

134 wms_id : `int` or `str`, optional 

135 Id or path that can be used by WMS service to look up job. 

136 user : `str`, optional 

137 User whose submitted jobs should be listed. 

138 require_bps : `bool`, optional 

139 Whether to require jobs returned in list to be bps-submitted jobs. 

140 pass_thru : `str`, optional 

141 Information to pass through to WMS. 

142 

143 Returns 

144 ------- 

145 job_ids : `list` [`Any`] 

146 Only job ids to be used by cancel and other functions. Typically 

147 this means top-level jobs (i.e., not children jobs). 

148 """ 

149 raise NotImplementedError 

150 

151 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None): 

152 """Query WMS for status of submitted WMS workflows. 

153 

154 Parameters 

155 ---------- 

156 wms_workflow_id : `int` or `str`, optional 

157 Id that can be used by WMS service to look up status. 

158 user : `str`, optional 

159 Limit report to submissions by this particular user. 

160 hist : `int`, optional 

161 Number of days to expand report to include finished WMS workflows. 

162 pass_thru : `str`, optional 

163 Additional arguments to pass through to the specific WMS service. 

164 

165 Returns 

166 ------- 

167 run_reports : `list` [`lsst.ctrl.bps.WmsRunReport`] 

168 Status information for submitted WMS workflows. 

169 message : `str` 

170 Message to user on how to find more status information specific to 

171 this particular WMS. 

172 """ 

173 raise NotImplementedError 

174 

175 def cancel(self, wms_id, pass_thru=None): 

176 """Cancel submitted workflows/jobs. 

177 

178 Parameters 

179 ---------- 

180 wms_id : `str` 

181 ID or path of job that should be canceled. 

182 pass_thru : `str`, optional 

183 Information to pass through to WMS. 

184 

185 Returns 

186 -------- 

187 deleted : `bool` 

188 Whether successful deletion or not. Currently, if any doubt or any 

189 individual jobs not deleted, return False. 

190 message : `str` 

191 Any message from WMS (e.g., error details). 

192 """ 

193 raise NotImplementedError 

194 

195 

196class BaseWmsWorkflow(metaclass=ABCMeta): 

197 """Interface for single workflow specific to a WMS. 

198 

199 Parameters 

200 ---------- 

201 name : `str` 

202 Unique name of workflow. 

203 config : `lsst.ctrl.bps.BpsConfig` 

204 Generic workflow config. 

205 """ 

206 def __init__(self, name, config): 

207 self.name = name 

208 self.config = config 

209 self.service_class = None 

210 self.run_id = None 

211 self.submit_path = None 

212 

213 @classmethod 

214 def from_generic_workflow(cls, config, generic_workflow, out_prefix, 

215 service_class): 

216 """Create a WMS-specific workflow from a GenericWorkflow 

217 

218 Parameters 

219 ---------- 

220 config : `lsst.ctrl.bps.BpsConfig` 

221 Configuration values needed for generating a WMS specific workflow. 

222 generic_workflow : `lsst.ctrl.bps.GenericWorkflow` 

223 Generic workflow from which to create the WMS-specific one. 

224 out_prefix : `str` 

225 Root directory to be used for WMS workflow inputs and outputs 

226 as well as internal WMS files. 

227 service_class : `str` 

228 Full module name of WMS service class that created this workflow. 

229 

230 Returns 

231 ------- 

232 wms_workflow : `lsst.ctrl.bps.BaseWmsWorkflow` 

233 A WMS specific workflow. 

234 """ 

235 

236 raise NotImplementedError 

237 

238 def write(self, out_prefix): 

239 """Write WMS files for this particular workflow. 

240 

241 Parameters 

242 ---------- 

243 out_prefix : `str` 

244 Root directory to be used for WMS workflow inputs and outputs 

245 as well as internal WMS files. 

246 """ 

247 raise NotImplementedError