Coverage for python/lsst/ctrl/bps/wms_service.py: 82%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Base classes for working with a specific WMS"""
25__all__ = ["BaseWmsService", "BaseWmsWorkflow", "WmsJobReport", "WmsRunReport", "WmsStates"]
28import logging
29import dataclasses
30from abc import ABCMeta
31from enum import Enum
34_LOG = logging.getLogger(__name__)
37class WmsStates(Enum):
38 """Run and job states
39 """
40 UNKNOWN = 0 # Can't determine state
41 MISFIT = 1 # Determined state, but doesn't fit other states
42 UNREADY = 2 # Still waiting for parents to finish
43 READY = 3 # All of its parents have finished successfully
44 PENDING = 4 # Ready to run, visible in batch queue
45 RUNNING = 5 # Currently running
46 DELETED = 6 # In the process of being deleted or already deleted
47 HELD = 7 # In a hold state
48 SUCCEEDED = 8 # Have completed with success status
49 FAILED = 9 # Have completed with non-success status
52@dataclasses.dataclass
53class WmsJobReport:
54 """WMS job information to be included in detailed report output
55 """
56 wms_id: str
57 name: str
58 label: str
59 state: WmsStates
61 __slots__ = ('wms_id', 'name', 'label', 'state')
64@dataclasses.dataclass
65class WmsRunReport:
66 """WMS run information to be included in detailed report output
67 """
68 wms_id: str
69 path: str
70 label: str
71 run: str
72 project: str
73 campaign: str
74 payload: str
75 operator: str
76 run_summary: str
77 state: WmsStates
78 jobs: list
79 total_number_jobs: int
80 job_state_counts: dict
82 __slots__ = ('wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload', 'operator',
83 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts')
86class BaseWmsService:
87 """Interface for interactions with a specific WMS.
89 Parameters
90 ----------
91 config : `lsst.ctrl.bps.BpsConfig`
92 Configuration needed by the WMS service.
93 """
94 def __init__(self, config):
95 self.config = config
97 def prepare(self, config, generic_workflow, out_prefix=None):
98 """Create submission for a generic workflow for a specific WMS.
100 Parameters
101 ----------
102 config : `lsst.ctrl.bps.BpsConfig`
103 BPS configuration.
104 generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
105 Generic representation of a single workflow
106 out_prefix : `str`
107 Prefix for all WMS output files
109 Returns
110 -------
111 wms_workflow : `BaseWmsWorkflow`
112 Prepared WMS Workflow to submit for execution
113 """
114 raise NotImplementedError
116 def submit(self, workflow):
117 """Submit a single WMS workflow
119 Parameters
120 ----------
121 workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
122 Prepared WMS Workflow to submit for execution
123 """
124 raise NotImplementedError
126 def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thru=None):
127 """Query WMS for list of submitted WMS workflows/jobs.
129 This should be a quick lookup function to create list of jobs for
130 other functions.
132 Parameters
133 ----------
134 wms_id : `int` or `str`, optional
135 Id or path that can be used by WMS service to look up job.
136 user : `str`, optional
137 User whose submitted jobs should be listed.
138 require_bps : `bool`, optional
139 Whether to require jobs returned in list to be bps-submitted jobs.
140 pass_thru : `str`, optional
141 Information to pass through to WMS.
143 Returns
144 -------
145 job_ids : `list` [`Any`]
146 Only job ids to be used by cancel and other functions. Typically
147 this means top-level jobs (i.e., not children jobs).
148 """
149 raise NotImplementedError
151 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None):
152 """Query WMS for status of submitted WMS workflows.
154 Parameters
155 ----------
156 wms_workflow_id : `int` or `str`, optional
157 Id that can be used by WMS service to look up status.
158 user : `str`, optional
159 Limit report to submissions by this particular user.
160 hist : `int`, optional
161 Number of days to expand report to include finished WMS workflows.
162 pass_thru : `str`, optional
163 Additional arguments to pass through to the specific WMS service.
165 Returns
166 -------
167 run_reports : `list` [`lsst.ctrl.bps.WmsRunReport`]
168 Status information for submitted WMS workflows.
169 message : `str`
170 Message to user on how to find more status information specific to
171 this particular WMS.
172 """
173 raise NotImplementedError
175 def cancel(self, wms_id, pass_thru=None):
176 """Cancel submitted workflows/jobs.
178 Parameters
179 ----------
180 wms_id : `str`
181 ID or path of job that should be canceled.
182 pass_thru : `str`, optional
183 Information to pass through to WMS.
185 Returns
186 --------
187 deleted : `bool`
188 Whether successful deletion or not. Currently, if any doubt or any
189 individual jobs not deleted, return False.
190 message : `str`
191 Any message from WMS (e.g., error details).
192 """
193 raise NotImplementedError
195 def run_submission_checks(self):
196 """Checks to run at start if running WMS specific submission steps.
198 Any exception other than NotImplementedError will halt submission.
199 Submit directory may not yet exist when this is called.
200 """
201 raise NotImplementedError
204class BaseWmsWorkflow(metaclass=ABCMeta):
205 """Interface for single workflow specific to a WMS.
207 Parameters
208 ----------
209 name : `str`
210 Unique name of workflow.
211 config : `lsst.ctrl.bps.BpsConfig`
212 Generic workflow config.
213 """
214 def __init__(self, name, config):
215 self.name = name
216 self.config = config
217 self.service_class = None
218 self.run_id = None
219 self.submit_path = None
221 @classmethod
222 def from_generic_workflow(cls, config, generic_workflow, out_prefix,
223 service_class):
224 """Create a WMS-specific workflow from a GenericWorkflow
226 Parameters
227 ----------
228 config : `lsst.ctrl.bps.BpsConfig`
229 Configuration values needed for generating a WMS specific workflow.
230 generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
231 Generic workflow from which to create the WMS-specific one.
232 out_prefix : `str`
233 Root directory to be used for WMS workflow inputs and outputs
234 as well as internal WMS files.
235 service_class : `str`
236 Full module name of WMS service class that created this workflow.
238 Returns
239 -------
240 wms_workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
241 A WMS specific workflow.
242 """
244 raise NotImplementedError
246 def write(self, out_prefix):
247 """Write WMS files for this particular workflow.
249 Parameters
250 ----------
251 out_prefix : `str`
252 Root directory to be used for WMS workflow inputs and outputs
253 as well as internal WMS files.
254 """
255 raise NotImplementedError