Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Base classes for working with a specific WMS""" 

23 

24import logging 

25import dataclasses 

26from abc import ABCMeta 

27from enum import Enum 

28 

29_LOG = logging.getLogger() 

30 

31 

32class WmsStates(Enum): 

33 """Run and job states 

34 """ 

35 UNKNOWN = 0 # Can't determine state 

36 MISFIT = 1 # Determined state, but doesn't fit other states 

37 UNREADY = 2 # Still waiting for parents to finish 

38 READY = 3 # All of its parents have finished successfully 

39 PENDING = 4 # Ready to run, visible in batch queue 

40 RUNNING = 5 # Currently running 

41 DELETED = 6 # In the process of being deleted or already deleted 

42 HELD = 7 # In a hold state 

43 SUCCEEDED = 8 # Have completed with success status 

44 FAILED = 9 # Have completed with non-success status 

45 

46 

47@dataclasses.dataclass 

48class WmsJobReport: 

49 """WMS job information to be included in detailed report output 

50 """ 

51 wms_id: str 

52 name: str 

53 label: str 

54 state: WmsStates 

55 

56 __slots__ = ('wms_id', 'name', 'label', 'state') 

57 

58 

59@dataclasses.dataclass 

60class WmsRunReport: 

61 """WMS run information to be included in detailed report output 

62 """ 

63 wms_id: str 

64 path: str 

65 label: str 

66 run: str 

67 project: str 

68 campaign: str 

69 payload: str 

70 operator: str 

71 run_summary: str 

72 state: WmsStates 

73 jobs: list 

74 total_number_jobs: int 

75 job_state_counts: dict 

76 

77 __slots__ = ('wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload', 'operator', 

78 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts') 

79 

80 

81class BaseWmsService: 

82 """Interface for interactions with a specific WMS. 

83 

84 Parameters 

85 ---------- 

86 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

87 Configuration needed by the WMS service. 

88 """ 

89 def __init__(self, config): 

90 self.config = config 

91 

92 def prepare(self, config, generic_workflow, out_prefix=None): 

93 """Create submission for a generic workflow for a specific WMS. 

94 

95 Parameters 

96 ---------- 

97 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

98 BPS configuration. 

99 generic_workflow : `~lsst.ctrl.bps.generic_workflow.GenericWorkflow` 

100 Generic representation of a single workflow 

101 out_prefix : `str` 

102 Prefix for all WMS output files 

103 

104 Returns 

105 ------- 

106 wms_workflow : `BaseWmsWorkflow` 

107 Prepared WMS Workflow to submit for execution 

108 """ 

109 raise NotImplementedError 

110 

111 def submit(self, workflow): 

112 """Submit a single WMS workflow 

113 

114 Parameters 

115 ---------- 

116 workflow : `~lsst.ctrl.bps.wms_service.BaseWmsWorkflow` 

117 Prepared WMS Workflow to submit for execution 

118 """ 

119 raise NotImplementedError 

120 

121 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None): 

122 """Query WMS for status of submitted WMS workflows. 

123 

124 Parameters 

125 ---------- 

126 wms_workflow_id : `int` or `str`, optional 

127 Id that can be used by WMS service to look up status. 

128 user : `str`, optional 

129 Limit report to submissions by this particular user. 

130 hist : `int`, optional 

131 Number of days to expand report to include finished WMS workflows. 

132 pass_thru : `str`, optional 

133 Additional arguments to pass through to the specific WMS service. 

134 

135 Returns 

136 ------- 

137 run_reports : `dict` of `~lsst.ctrl.bps.wms_service.BaseWmsReport` 

138 Status information for submitted WMS workflows. 

139 message : `str` 

140 Message to user on how to find more status information specific to 

141 this particular WMS. 

142 """ 

143 raise NotImplementedError 

144 

145 

146class BaseWmsWorkflow(metaclass=ABCMeta): 

147 """Interface for single workflow specific to a WMS. 

148 

149 Parameters 

150 ---------- 

151 name : `str` 

152 Unique name of workflow. 

153 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

154 Generic workflow config. 

155 """ 

156 def __init__(self, name, config): 

157 self.name = name 

158 self.config = config 

159 self.service_class = None 

160 self.run_id = None 

161 self.submit_path = None 

162 

163 @classmethod 

164 def from_generic_workflow(cls, config, generic_workflow, out_prefix, 

165 service_class): 

166 """Create a WMS-specific workflow from a GenericWorkflow 

167 

168 Parameters 

169 ---------- 

170 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

171 Configuration values needed for generating a WMS specific workflow. 

172 generic_workflow : `~lsst.ctrl.bps.generic_workflow.GenericWorkflow` 

173 Generic workflow from which to create the WMS-specific one. 

174 out_prefix : `str` 

175 Root directory to be used for WMS workflow inputs and outputs 

176 as well as internal WMS files. 

177 service_class : `str` 

178 Full module name of WMS service class that created this workflow. 

179 

180 Returns 

181 ------- 

182 wms_workflow : `~lsst.ctrl.bps.wms_service.BaseWmsWorkflow` 

183 A WMS specific workflow 

184 """ 

185 

186 raise NotImplementedError 

187 

188 def write(self, out_prefix): 

189 """Write WMS files for this particular workflow. 

190 

191 Parameters 

192 ---------- 

193 out_prefix : `str` 

194 Root directory to be used for WMS workflow inputs and outputs 

195 as well as internal WMS files. 

196 """ 

197 raise NotImplementedError