Coverage for python/lsst/ctrl/bps/wms_service.py: 82%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Base classes for working with a specific WMS"""
25__all__ = ["BaseWmsService", "BaseWmsWorkflow", "WmsJobReport", "WmsRunReport", "WmsStates"]
28import logging
29import dataclasses
30from abc import ABCMeta
31from enum import Enum
34_LOG = logging.getLogger(__name__)
37class WmsStates(Enum):
38 """Run and job states
39 """
40 UNKNOWN = 0 # Can't determine state
41 MISFIT = 1 # Determined state, but doesn't fit other states
42 UNREADY = 2 # Still waiting for parents to finish
43 READY = 3 # All of its parents have finished successfully
44 PENDING = 4 # Ready to run, visible in batch queue
45 RUNNING = 5 # Currently running
46 DELETED = 6 # In the process of being deleted or already deleted
47 HELD = 7 # In a hold state
48 SUCCEEDED = 8 # Have completed with success status
49 FAILED = 9 # Have completed with non-success status
52@dataclasses.dataclass
53class WmsJobReport:
54 """WMS job information to be included in detailed report output
55 """
56 wms_id: str
57 name: str
58 label: str
59 state: WmsStates
61 __slots__ = ('wms_id', 'name', 'label', 'state')
64@dataclasses.dataclass
65class WmsRunReport:
66 """WMS run information to be included in detailed report output
67 """
68 wms_id: str
69 global_wms_id: str
70 path: str
71 label: str
72 run: str
73 project: str
74 campaign: str
75 payload: str
76 operator: str
77 run_summary: str
78 state: WmsStates
79 jobs: list
80 total_number_jobs: int
81 job_state_counts: dict
83 __slots__ = ('wms_id', 'global_wms_id', 'path', 'label', 'run', 'project', 'campaign', 'payload',
84 'operator', 'run_summary', 'state', 'total_number_jobs', 'jobs', 'job_state_counts')
87class BaseWmsService:
88 """Interface for interactions with a specific WMS.
90 Parameters
91 ----------
92 config : `lsst.ctrl.bps.BpsConfig`
93 Configuration needed by the WMS service.
94 """
95 def __init__(self, config):
96 self.config = config
98 def prepare(self, config, generic_workflow, out_prefix=None):
99 """Create submission for a generic workflow for a specific WMS.
101 Parameters
102 ----------
103 config : `lsst.ctrl.bps.BpsConfig`
104 BPS configuration.
105 generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
106 Generic representation of a single workflow
107 out_prefix : `str`
108 Prefix for all WMS output files
110 Returns
111 -------
112 wms_workflow : `BaseWmsWorkflow`
113 Prepared WMS Workflow to submit for execution
114 """
115 raise NotImplementedError
117 def submit(self, workflow):
118 """Submit a single WMS workflow
120 Parameters
121 ----------
122 workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
123 Prepared WMS Workflow to submit for execution
124 """
125 raise NotImplementedError
127 def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thru=None, is_global=False):
128 """Query WMS for list of submitted WMS workflows/jobs.
130 This should be a quick lookup function to create list of jobs for
131 other functions.
133 Parameters
134 ----------
135 wms_id : `int` or `str`, optional
136 Id or path that can be used by WMS service to look up job.
137 user : `str`, optional
138 User whose submitted jobs should be listed.
139 require_bps : `bool`, optional
140 Whether to require jobs returned in list to be bps-submitted jobs.
141 pass_thru : `str`, optional
142 Information to pass through to WMS.
143 is_global : `bool`, optional
144 If set, all available job queues will be queried for job
145 information. Defaults to False which means that only a local job
146 queue will be queried for information.
148 Only applicable in the context of a WMS using distributed job
149 queues (e.g., HTCondor). A WMS with a centralized job queue
150 (e.g. PanDA) can safely ignore it.
152 Returns
153 -------
154 job_ids : `list` [`Any`]
155 Only job ids to be used by cancel and other functions. Typically
156 this means top-level jobs (i.e., not children jobs).
157 """
158 raise NotImplementedError
160 def report(self, wms_workflow_id=None, user=None, hist=0, pass_thru=None, is_global=False):
161 """Query WMS for status of submitted WMS workflows.
163 Parameters
164 ----------
165 wms_workflow_id : `int` or `str`, optional
166 Id that can be used by WMS service to look up status.
167 user : `str`, optional
168 Limit report to submissions by this particular user.
169 hist : `int`, optional
170 Number of days to expand report to include finished WMS workflows.
171 pass_thru : `str`, optional
172 Additional arguments to pass through to the specific WMS service.
173 is_global : `bool`, optional
174 If set, all available job queues will be queried for job
175 information. Defaults to False which means that only a local job
176 queue will be queried for information.
178 Only applicable in the context of a WMS using distributed job
179 queues (e.g., HTCondor). A WMS with a centralized job queue
180 (e.g. PanDA) can safely ignore it.
182 Returns
183 -------
184 run_reports : `list` [`lsst.ctrl.bps.WmsRunReport`]
185 Status information for submitted WMS workflows.
186 message : `str`
187 Message to user on how to find more status information specific to
188 this particular WMS.
189 """
190 raise NotImplementedError
192 def cancel(self, wms_id, pass_thru=None):
193 """Cancel submitted workflows/jobs.
195 Parameters
196 ----------
197 wms_id : `str`
198 ID or path of job that should be canceled.
199 pass_thru : `str`, optional
200 Information to pass through to WMS.
202 Returns
203 --------
204 deleted : `bool`
205 Whether successful deletion or not. Currently, if any doubt or any
206 individual jobs not deleted, return False.
207 message : `str`
208 Any message from WMS (e.g., error details).
209 """
210 raise NotImplementedError
213class BaseWmsWorkflow(metaclass=ABCMeta):
214 """Interface for single workflow specific to a WMS.
216 Parameters
217 ----------
218 name : `str`
219 Unique name of workflow.
220 config : `lsst.ctrl.bps.BpsConfig`
221 Generic workflow config.
222 """
223 def __init__(self, name, config):
224 self.name = name
225 self.config = config
226 self.service_class = None
227 self.run_id = None
228 self.submit_path = None
230 @classmethod
231 def from_generic_workflow(cls, config, generic_workflow, out_prefix,
232 service_class):
233 """Create a WMS-specific workflow from a GenericWorkflow
235 Parameters
236 ----------
237 config : `lsst.ctrl.bps.BpsConfig`
238 Configuration values needed for generating a WMS specific workflow.
239 generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
240 Generic workflow from which to create the WMS-specific one.
241 out_prefix : `str`
242 Root directory to be used for WMS workflow inputs and outputs
243 as well as internal WMS files.
244 service_class : `str`
245 Full module name of WMS service class that created this workflow.
247 Returns
248 -------
249 wms_workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
250 A WMS specific workflow.
251 """
253 raise NotImplementedError
255 def write(self, out_prefix):
256 """Write WMS files for this particular workflow.
258 Parameters
259 ----------
260 out_prefix : `str`
261 Root directory to be used for WMS workflow inputs and outputs
262 as well as internal WMS files.
263 """
264 raise NotImplementedError