Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Base classes for working with a specific WMS""" 

23 

24import logging 

25import dataclasses 

26from abc import ABCMeta 

27from enum import Enum 

28 

29_LOG = logging.getLogger(__name__) 

30 

31 

32class WmsStates(Enum): 

33 """Run and job states 

34 """ 

35 UNKNOWN = 0 # Can't determine state 

36 MISFIT = 1 # Determined state, but doesn't fit other states 

37 UNREADY = 2 # Still waiting for parents to finish 

38 READY = 3 # All of its parents have finished successfully 

39 PENDING = 4 # Ready to run, visible in batch queue 

40 RUNNING = 5 # Currently running 

41 DELETED = 6 # In the process of being deleted or already deleted 

42 HELD = 7 # In a hold state 

43 SUCCEEDED = 8 # Have completed with success status 

44 FAILED = 9 # Have completed with non-success status 

45 

46 

47@dataclasses.dataclass 

48class WmsJobReport: 

49 """WMS job information to be included in detailed report output 

50 """ 

51 wms_id: str 

52 name: str 

53 label: str 

54 state: WmsStates 

55 

56 __slots__ = ('wms_id', 'name', 'label', 'state') 

57 

58 

59@dataclasses.dataclass 

60class WmsRunReport: 

61 """WMS run information to be included in detailed report output 

62 """ 

63 wms_id: str 

64 path: str 

65 label: str 

66 run: str 

67 project: str 

68 campaign: str 

69 payload: str 

70 operator: str 

71 run_summary: str 

72 state: WmsStates 

73 jobs: list 

74 total_number_jobs: int 

75 job_state_counts: dict 

76 

77 __slots__ = ('wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload', 'operator', 

78 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts') 

79 

80 

81class BaseWmsService: 

82 """Interface for interactions with a specific WMS. 

83 

84 Parameters 

85 ---------- 

86 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

87 Configuration needed by the WMS service. 

88 """ 

89 def __init__(self, config): 

90 self.config = config 

91 

92 def prepare(self, config, generic_workflow, out_prefix=None): 

93 """Create submission for a generic workflow for a specific WMS. 

94 

95 Parameters 

96 ---------- 

97 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

98 BPS configuration. 

99 generic_workflow : `~lsst.ctrl.bps.generic_workflow.GenericWorkflow` 

100 Generic representation of a single workflow 

101 out_prefix : `str` 

102 Prefix for all WMS output files 

103 

104 Returns 

105 ------- 

106 wms_workflow : `BaseWmsWorkflow` 

107 Prepared WMS Workflow to submit for execution 

108 """ 

109 raise NotImplementedError 

110 

111 def submit(self, workflow): 

112 """Submit a single WMS workflow 

113 

114 Parameters 

115 ---------- 

116 workflow : `~lsst.ctrl.bps.wms_service.BaseWmsWorkflow` 

117 Prepared WMS Workflow to submit for execution 

118 """ 

119 raise NotImplementedError 

120 

121 def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thru=None): 

122 """Query WMS for list of submitted WMS workflows/jobs. 

123 

124 This should be a quick lookup function to create list of jobs for 

125 other functions. 

126 

127 Parameters 

128 ---------- 

129 wms_id : `int` or `str`, optional 

130 Id or path that can be used by WMS service to look up job. 

131 user : `str`, optional 

132 User whose submitted jobs should be listed. 

133 require_bps : `bool`, optional 

134 Whether to require jobs returned in list to be bps-submitted jobs. 

135 pass_thru : `str`, optional 

136 Information to pass through to WMS. 

137 

138 Returns 

139 ------- 

140 job_ids : `list` of `Any` 

141 Only job ids to be used by cancel and other functions. Typically 

142 this means top-level jobs (i.e., not children jobs). 

143 """ 

144 raise NotImplementedError 

145 

146 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None): 

147 """Query WMS for status of submitted WMS workflows. 

148 

149 Parameters 

150 ---------- 

151 wms_workflow_id : `int` or `str`, optional 

152 Id that can be used by WMS service to look up status. 

153 user : `str`, optional 

154 Limit report to submissions by this particular user. 

155 hist : `int`, optional 

156 Number of days to expand report to include finished WMS workflows. 

157 pass_thru : `str`, optional 

158 Additional arguments to pass through to the specific WMS service. 

159 

160 Returns 

161 ------- 

162 run_reports : `dict` of `~lsst.ctrl.bps.wms_service.BaseWmsReport` 

163 Status information for submitted WMS workflows. 

164 message : `str` 

165 Message to user on how to find more status information specific to 

166 this particular WMS. 

167 """ 

168 raise NotImplementedError 

169 

170 def cancel(self, wms_id, pass_thru=None): 

171 """Cancel submitted workflows/jobs. 

172 

173 Parameters 

174 ---------- 

175 wms_id : `str` 

176 ID or path of job that should be canceled. 

177 pass_thru : `str`, optional 

178 Information to pass through to WMS. 

179 

180 Returns 

181 -------- 

182 deleted : `bool` 

183 Whether successful deletion or not. Currently, if any doubt or any 

184 individual jobs not deleted, return False. 

185 message : `str` 

186 Any message from WMS (e.g., error details). 

187 """ 

188 raise NotImplementedError 

189 

190 

191class BaseWmsWorkflow(metaclass=ABCMeta): 

192 """Interface for single workflow specific to a WMS. 

193 

194 Parameters 

195 ---------- 

196 name : `str` 

197 Unique name of workflow. 

198 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

199 Generic workflow config. 

200 """ 

201 def __init__(self, name, config): 

202 self.name = name 

203 self.config = config 

204 self.service_class = None 

205 self.run_id = None 

206 self.submit_path = None 

207 

208 @classmethod 

209 def from_generic_workflow(cls, config, generic_workflow, out_prefix, 

210 service_class): 

211 """Create a WMS-specific workflow from a GenericWorkflow 

212 

213 Parameters 

214 ---------- 

215 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

216 Configuration values needed for generating a WMS specific workflow. 

217 generic_workflow : `~lsst.ctrl.bps.generic_workflow.GenericWorkflow` 

218 Generic workflow from which to create the WMS-specific one. 

219 out_prefix : `str` 

220 Root directory to be used for WMS workflow inputs and outputs 

221 as well as internal WMS files. 

222 service_class : `str` 

223 Full module name of WMS service class that created this workflow. 

224 

225 Returns 

226 ------- 

227 wms_workflow : `~lsst.ctrl.bps.wms_service.BaseWmsWorkflow` 

228 A WMS specific workflow 

229 """ 

230 

231 raise NotImplementedError 

232 

233 def write(self, out_prefix): 

234 """Write WMS files for this particular workflow. 

235 

236 Parameters 

237 ---------- 

238 out_prefix : `str` 

239 Root directory to be used for WMS workflow inputs and outputs 

240 as well as internal WMS files. 

241 """ 

242 raise NotImplementedError