Coverage for python/lsst/ctrl/bps/wms_service.py: 81%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Base classes for working with a specific WMS"""
25__all__ = ["BaseWmsService", "BaseWmsWorkflow", "WmsJobReport", "WmsRunReport", "WmsStates"]
28import logging
29import dataclasses
30from abc import ABCMeta
31from enum import Enum
34_LOG = logging.getLogger(__name__)
37class WmsStates(Enum):
38 """Run and job states
39 """
40 UNKNOWN = 0 # Can't determine state
41 MISFIT = 1 # Determined state, but doesn't fit other states
42 UNREADY = 2 # Still waiting for parents to finish
43 READY = 3 # All of its parents have finished successfully
44 PENDING = 4 # Ready to run, visible in batch queue
45 RUNNING = 5 # Currently running
46 DELETED = 6 # In the process of being deleted or already deleted
47 HELD = 7 # In a hold state
48 SUCCEEDED = 8 # Have completed with success status
49 FAILED = 9 # Have completed with non-success status
52@dataclasses.dataclass
53class WmsJobReport:
54 """WMS job information to be included in detailed report output
55 """
56 wms_id: str
57 name: str
58 label: str
59 state: WmsStates
61 __slots__ = ('wms_id', 'name', 'label', 'state')
64@dataclasses.dataclass
65class WmsRunReport:
66 """WMS run information to be included in detailed report output
67 """
68 wms_id: str
69 global_wms_id: str
70 path: str
71 label: str
72 run: str
73 project: str
74 campaign: str
75 payload: str
76 operator: str
77 run_summary: str
78 state: WmsStates
79 jobs: list
80 total_number_jobs: int
81 job_state_counts: dict
83 __slots__ = ('wms_id', 'global_wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload',
84 'operator', 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts')
87class BaseWmsService:
88 """Interface for interactions with a specific WMS.
90 Parameters
91 ----------
92 config : `lsst.ctrl.bps.BpsConfig`
93 Configuration needed by the WMS service.
94 """
95 def __init__(self, config):
96 self.config = config
98 def prepare(self, config, generic_workflow, out_prefix=None):
99 """Create submission for a generic workflow for a specific WMS.
101 Parameters
102 ----------
103 config : `lsst.ctrl.bps.BpsConfig`
104 BPS configuration.
105 generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
106 Generic representation of a single workflow
107 out_prefix : `str`
108 Prefix for all WMS output files
110 Returns
111 -------
112 wms_workflow : `BaseWmsWorkflow`
113 Prepared WMS Workflow to submit for execution
114 """
115 raise NotImplementedError
117 def submit(self, workflow):
118 """Submit a single WMS workflow
120 Parameters
121 ----------
122 workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
123 Prepared WMS Workflow to submit for execution
124 """
125 raise NotImplementedError
127 def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thru=None, is_global=False):
128 """Query WMS for list of submitted WMS workflows/jobs.
130 This should be a quick lookup function to create list of jobs for
131 other functions.
133 Parameters
134 ----------
135 wms_id : `int` or `str`, optional
136 Id or path that can be used by WMS service to look up job.
137 user : `str`, optional
138 User whose submitted jobs should be listed.
139 require_bps : `bool`, optional
140 Whether to require jobs returned in list to be bps-submitted jobs.
141 pass_thru : `str`, optional
142 Information to pass through to WMS.
143 is_global : `bool`, optional
144 If set, all available job queues will be queried for job
145 information. Defaults to False which means that only a local job
146 queue will be queried for information.
148 Only applicable in the context of a WMS using distributed job
149 queues (e.g., HTCondor). A WMS with a centralized job queue
150 (e.g. PanDA) can safely ignore it.
152 Returns
153 -------
154 job_ids : `list` [`Any`]
155 Only job ids to be used by cancel and other functions. Typically
156 this means top-level jobs (i.e., not children jobs).
157 """
158 raise NotImplementedError
160 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None, is_global=False):
161 """Query WMS for status of submitted WMS workflows.
163 Parameters
164 ----------
165 wms_workflow_id : `int` or `str`, optional
166 Id that can be used by WMS service to look up status.
167 user : `str`, optional
168 Limit report to submissions by this particular user.
169 hist : `int`, optional
170 Number of days to expand report to include finished WMS workflows.
171 pass_thru : `str`, optional
172 Additional arguments to pass through to the specific WMS service.
173 is_global : `bool`, optional
174 If set, all available job queues will be queried for job
175 information. Defaults to False which means that only a local job
176 queue will be queried for information.
178 Only applicable in the context of a WMS using distributed job
179 queues (e.g., HTCondor). A WMS with a centralized job queue
180 (e.g. PanDA) can safely ignore it.
182 Returns
183 -------
184 run_reports : `list` [`lsst.ctrl.bps.WmsRunReport`]
185 Status information for submitted WMS workflows.
186 message : `str`
187 Message to user on how to find more status information specific to
188 this particular WMS.
189 """
190 raise NotImplementedError
192 def cancel(self, wms_id, pass_thru=None):
193 """Cancel submitted workflows/jobs.
195 Parameters
196 ----------
197 wms_id : `str`
198 ID or path of job that should be canceled.
199 pass_thru : `str`, optional
200 Information to pass through to WMS.
202 Returns
203 --------
204 deleted : `bool`
205 Whether successful deletion or not. Currently, if any doubt or any
206 individual jobs not deleted, return False.
207 message : `str`
208 Any message from WMS (e.g., error details).
209 """
210 raise NotImplementedError
212 def run_submission_checks(self):
213 """Checks to run at start if running WMS specific submission steps.
215 Any exception other than NotImplementedError will halt submission.
216 Submit directory may not yet exist when this is called.
217 """
218 raise NotImplementedError
221class BaseWmsWorkflow(metaclass=ABCMeta):
222 """Interface for single workflow specific to a WMS.
224 Parameters
225 ----------
226 name : `str`
227 Unique name of workflow.
228 config : `lsst.ctrl.bps.BpsConfig`
229 Generic workflow config.
230 """
231 def __init__(self, name, config):
232 self.name = name
233 self.config = config
234 self.service_class = None
235 self.run_id = None
236 self.submit_path = None
238 @classmethod
239 def from_generic_workflow(cls, config, generic_workflow, out_prefix,
240 service_class):
241 """Create a WMS-specific workflow from a GenericWorkflow
243 Parameters
244 ----------
245 config : `lsst.ctrl.bps.BpsConfig`
246 Configuration values needed for generating a WMS specific workflow.
247 generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
248 Generic workflow from which to create the WMS-specific one.
249 out_prefix : `str`
250 Root directory to be used for WMS workflow inputs and outputs
251 as well as internal WMS files.
252 service_class : `str`
253 Full module name of WMS service class that created this workflow.
255 Returns
256 -------
257 wms_workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
258 A WMS specific workflow.
259 """
261 raise NotImplementedError
263 def write(self, out_prefix):
264 """Write WMS files for this particular workflow.
266 Parameters
267 ----------
268 out_prefix : `str`
269 Root directory to be used for WMS workflow inputs and outputs
270 as well as internal WMS files.
271 """
272 raise NotImplementedError