Coverage for python/lsst/ctrl/bps/drivers.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Driver functions for each subcommand.
24Driver functions ensure that ensure all setup work is done before running
25the subcommand method.
26"""
29__all__ = [
30 "acquire_qgraph_driver",
31 "cluster_qgraph_driver",
32 "transform_driver",
33 "prepare_driver",
34 "submit_driver",
35 "report_driver",
36 "cancel_driver",
37]
40import getpass
41import logging
42import os
43import pickle
44import time
46from lsst.obs.base import Instrument
48from . import BPS_SEARCH_ORDER, BpsConfig
49from .bps_draw import draw_networkx_dot
50from .pre_transform import acquire_quantum_graph, cluster_quanta
51from .transform import transform
52from .prepare import prepare
53from .submit import submit
54from .cancel import cancel
55from .report import report
58_LOG = logging.getLogger(__name__)
61def _init_submission_driver(config_file):
62 """Initialize runtime environment.
64 Parameters
65 ----------
66 config_file : `str`
67 Name of the configuration file.
69 Returns
70 -------
71 config : `lsst.ctrl.bps.BpsConfig`
72 Batch Processing Service configuration.
73 """
74 config = BpsConfig(config_file, BPS_SEARCH_ORDER)
75 config[".bps_defined.timestamp"] = Instrument.makeCollectionTimestamp()
76 if "uniqProcName" not in config:
77 config[".bps_defined.uniqProcName"] = config["outCollection"].replace("/", "_")
78 if "operator" not in config:
79 config[".bps_defined.operator"] = getpass.getuser()
81 # make submit directory to contain all outputs
82 submit_path = config["submitPath"]
83 os.makedirs(submit_path, exist_ok=True)
84 config[".bps_defined.submit_path"] = submit_path
85 return config
88def acquire_qgraph_driver(config_file):
89 """Read a quantum graph from a file or create one from pipeline definition.
91 Parameters
92 ----------
93 config_file : `str`
94 Name of the configuration file.
96 Returns
97 -------
98 config : `lsst.ctrl.bps.BpsConfig`
99 Updated configuration.
100 qgraph : `lsst.pipe.base.graph.QuantumGraph`
101 A graph representing quanta.
102 """
103 stime = time.time()
104 config = _init_submission_driver(config_file)
105 submit_path = config[".bps_defined.submit_path"]
106 _LOG.info("Acquiring QuantumGraph (it will be created from pipeline definition if needed)")
107 qgraph_file, qgraph = acquire_quantum_graph(config, out_prefix=submit_path)
108 _LOG.info("Run QuantumGraph file %s", qgraph_file)
109 config[".bps_defined.run_qgraph_file"] = qgraph_file
110 _LOG.info("Acquiring QuantumGraph took %.2f seconds", time.time() - stime)
111 return config, qgraph
114def cluster_qgraph_driver(config_file):
115 """Group quanta into clusters.
117 Parameters
118 ----------
119 config_file : `str`
120 Name of the configuration file.
122 Returns
123 -------
124 config : `lsst.ctrl.bps.BpsConfig`
125 Updated configuration.
126 clustered_qgraph : `lsst.ctrl.bps.ClusteredQuantumGraph`
127 A graph representing clustered quanta.
128 """
129 stime = time.time()
130 config, qgraph = acquire_qgraph_driver(config_file)
131 _LOG.info("Clustering quanta")
132 clustered_qgraph = cluster_quanta(config, qgraph, config["uniqProcName"])
133 _LOG.info("Clustering quanta took %.2f seconds", time.time() - stime)
135 submit_path = config[".bps_defined.submit_path"]
136 _, save_clustered_qgraph = config.search("saveClusteredQgraph", opt={"default": False})
137 if save_clustered_qgraph:
138 with open(os.path.join(submit_path, "bps_clustered_qgraph.pickle"), "wb") as outfh:
139 pickle.dump(clustered_qgraph, outfh)
140 _, save_dot = config.search("saveDot", opt={"default": False})
141 if save_dot:
142 draw_networkx_dot(clustered_qgraph, os.path.join(submit_path, "bps_clustered_qgraph.dot"))
143 return config, clustered_qgraph
146def transform_driver(config_file):
147 """Create a workflow for a specific workflow management system.
149 Parameters
150 ----------
151 config_file : `str`
152 Name of the configuration file.
154 Returns
155 -------
156 generic_workflow_config : `lsst.ctrl.bps.BpsConfig`
157 Configuration to use when creating the workflow.
158 generic_workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
159 Representation of the abstract/scientific workflow specific to a given
160 workflow management system.
161 """
162 stime = time.time()
163 config, clustered_qgraph = cluster_qgraph_driver(config_file)
164 submit_path = config[".bps_defined.submit_path"]
165 _LOG.info("Creating Generic Workflow")
166 generic_workflow, generic_workflow_config = transform(config, clustered_qgraph, submit_path)
167 _LOG.info("Creating Generic Workflow took %.2f seconds", time.time() - stime)
168 _LOG.info("Generic Workflow name %s", generic_workflow.name)
170 _, save_workflow = config.search("saveGenericWorkflow", opt={"default": False})
171 if save_workflow:
172 with open(os.path.join(submit_path, "bps_generic_workflow.pickle"), "wb") as outfh:
173 generic_workflow.save(outfh, "pickle")
174 _, save_dot = config.search("saveDot", opt={"default": False})
175 if save_dot:
176 with open(os.path.join(submit_path, "bps_generic_workflow.dot"), "w") as outfh:
177 generic_workflow.draw(outfh, "dot")
178 return generic_workflow_config, generic_workflow
181def prepare_driver(config_file):
182 """Create a representation of the generic workflow.
184 Parameters
185 ----------
186 config_file : `str`
187 Name of the configuration file.
189 Returns
190 -------
191 wms_config : `lsst.ctrl.bps.BpsConfig`
192 Configuration to use when creating the workflow.
193 workflow : `lsst.ctrl.bps.BaseWmsWorkflow`
194 Representation of the abstract/scientific workflow specific to a given
195 workflow management system.
196 """
197 stime = time.time()
198 generic_workflow_config, generic_workflow = transform_driver(config_file)
199 submit_path = generic_workflow_config[".bps_defined.submit_path"]
200 _LOG.info("Creating specific implementation of workflow")
201 wms_workflow = prepare(generic_workflow_config, generic_workflow, submit_path)
202 wms_workflow_config = generic_workflow_config
203 _LOG.info("Creating specific implementation of workflow took %.2f seconds", time.time() - stime)
204 print(f"Submit dir: {wms_workflow.submit_path}")
205 return wms_workflow_config, wms_workflow
208def submit_driver(config_file):
209 """Submit workflow for execution.
211 Parameters
212 ----------
213 config_file : `str`
214 Name of the configuration file.
215 """
216 wms_workflow_config, wms_workflow = prepare_driver(config_file)
217 submit(wms_workflow_config, wms_workflow)
218 print(f"Run Id: {wms_workflow.run_id}")
221def report_driver(wms_service, run_id, user, hist_days, pass_thru):
222 """Print out summary of jobs submitted for execution.
224 Parameters
225 ----------
226 wms_service : `str`
227 Name of the class.
228 run_id : `str`
229 A run id the report will be restricted to.
230 user : `str`
231 A user name the report will be restricted to.
232 hist_days : int
233 Number of days
234 pass_thru : `str`
235 A string to pass directly to the WMS service class.
236 """
237 report(wms_service, run_id, user, hist_days, pass_thru)
240def cancel_driver(wms_service, run_id, user, require_bps, pass_thru):
241 """Cancel submitted workflows.
243 Parameters
244 ----------
245 wms_service : `str`
246 Name of the Workload Management System service class.
247 run_id : `str`
248 ID or path of job that should be canceled.
249 user : `str`
250 User whose submitted jobs should be canceled.
251 require_bps : `bool`
252 Whether to require given run_id/user to be a bps submitted job.
253 pass_thru : `str`
254 Information to pass through to WMS.
255 """
256 cancel(wms_service, run_id, user, require_bps, pass_thru)