Coverage for python/lsst/ctrl/bps/drivers.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21"""Driver functions for each subcommand.
23Driver functions ensure that ensure all setup work is done before running
24the subcommand method.
25"""
27__all__ = [
28 "transform_driver",
29 "prepare_driver",
30 "submit_driver",
31 "report_driver",
32 "cancel_driver",
33]
35import getpass
36import logging
37import os
38import pickle
39import time
41from lsst.obs.base import Instrument
43from . import BpsConfig
44from .bps_draw import draw_networkx_dot
45from .pre_transform import acquire_quantum_graph, cluster_quanta
46from .transform import transform
47from .prepare import prepare
48from .submit import BPS_SEARCH_ORDER, submit
49from .cancel import cancel
50from .report import report
53_LOG = logging.getLogger(__name__)
56def _init_submission_driver(config_file):
57 """Initialize runtime environment.
59 Parameters
60 ----------
61 config_file : `str`
62 Name of the configuration file.
64 Returns
65 -------
66 config : `~lsst.ctrl.bps.BpsConfig`
67 Batch Processing Service configuration.
68 """
69 config = BpsConfig(config_file, BPS_SEARCH_ORDER)
70 config[".bps_defined.timestamp"] = Instrument.makeCollectionTimestamp()
71 if "uniqProcName" not in config:
72 config[".bps_defined.uniqProcName"] = config["outCollection"].replace("/", "_")
73 if "operator" not in config:
74 config[".bps_defined.operator"] = getpass.getuser()
76 # make submit directory to contain all outputs
77 submit_path = config["submitPath"]
78 os.makedirs(submit_path, exist_ok=True)
79 config[".bps_defined.submit_path"] = submit_path
80 return config
83def acquire_qgraph_driver(config_file):
84 """Read a quantum graph from a file or create one from pipeline definition.
86 Parameters
87 ----------
88 config_file : `str`
89 Name of the configuration file.
91 Returns
92 -------
93 config : `~lsst.ctrl.bps.BpsConfig`
94 Updated configuration.
95 qgraph : `~lsst.pipe.base.graph.QuantumGraph`
96 A graph representing quanta.
97 """
98 stime = time.time()
99 config = _init_submission_driver(config_file)
100 submit_path = config[".bps_defined.submit_path"]
101 _LOG.info("Acquiring QuantumGraph (it will be created from pipeline definition if needed)")
102 qgraph_file, qgraph = acquire_quantum_graph(config, out_prefix=submit_path)
103 _LOG.info("Run QuantumGraph file %s", qgraph_file)
104 config[".bps_defined.run_qgraph_file"] = qgraph_file
105 _LOG.info("Acquiring QuantumGraph took %.2f seconds", time.time() - stime)
106 return config, qgraph
109def cluster_qgraph_driver(config_file):
110 """Group quanta into clusters.
112 Parameters
113 ----------
114 config_file : `str`
115 Name of the configuration file.
117 Returns
118 -------
119 config : `~lsst.ctrl.bps.BpsConfig`
120 Updated configuration.
121 clustered_qgraph : `~lsst.ctrl.bps.ClusteredQuantumGraph`
122 A graph representing clustered quanta.
123 """
124 stime = time.time()
125 config, qgraph = acquire_qgraph_driver(config_file)
126 _LOG.info("Clustering quanta")
127 clustered_qgraph = cluster_quanta(config, qgraph, config["uniqProcName"])
128 _LOG.info("Clustering quanta took %.2f seconds", time.time() - stime)
130 submit_path = config[".bps_defined.submit_path"]
131 _, save_clustered_qgraph = config.search("saveClusteredQgraph", opt={"default": False})
132 if save_clustered_qgraph:
133 with open(os.path.join(submit_path, "bps_clustered_qgraph.pickle"), "wb") as outfh:
134 pickle.dump(clustered_qgraph, outfh)
135 _, save_dot = config.search("saveDot", opt={"default": False})
136 if save_dot:
137 draw_networkx_dot(clustered_qgraph, os.path.join(submit_path, "bps_clustered_qgraph.dot"))
138 return config, clustered_qgraph
141def transform_driver(config_file):
142 """Create a workflow for a specific workflow management system.
144 Parameters
145 ----------
146 config_file : `str`
147 Name of the configuration file.
149 Returns
150 -------
151 generic_workflow_config : `lsst.ctrl.bps.BpsConfig`
152 Configuration to use when creating the workflow.
153 generic_workflow : `lsst.ctrl.bps.wms_workflow.BaseWmsWorkflow`
154 Representation of the abstract/scientific workflow specific to a given
155 workflow management system.
156 """
157 stime = time.time()
158 config, clustered_qgraph = cluster_qgraph_driver(config_file)
159 submit_path = config[".bps_defined.submit_path"]
160 _LOG.info("Creating Generic Workflow")
161 generic_workflow, generic_workflow_config = transform(config, clustered_qgraph, submit_path)
162 _LOG.info("Creating Generic Workflow took %.2f seconds", time.time() - stime)
163 _LOG.info("Generic Workflow name %s", generic_workflow.name)
165 _, save_workflow = config.search("saveGenericWorkflow", opt={"default": False})
166 if save_workflow:
167 with open(os.path.join(submit_path, "bps_generic_workflow.pickle"), "wb") as outfh:
168 generic_workflow.save(outfh, "pickle")
169 _, save_dot = config.search("saveDot", opt={"default": False})
170 if save_dot:
171 with open(os.path.join(submit_path, "bps_generic_workflow.dot"), "w") as outfh:
172 generic_workflow.draw(outfh, "dot")
173 return generic_workflow_config, generic_workflow
176def prepare_driver(config_file):
177 """Create a representation of the generic workflow.
179 Parameters
180 ----------
181 config_file : `str`
182 Name of the configuration file.
184 Returns
185 -------
186 wms_config : `lsst.ctrl.bps.BpsConfig`
187 Configuration to use when creating the workflow.
188 workflow : `lsst.ctrl.bps.wms_workflow.BaseWmsWorkflow`
189 Representation of the abstract/scientific workflow specific to a given
190 workflow management system.
191 """
192 stime = time.time()
193 generic_workflow_config, generic_workflow = transform_driver(config_file)
194 submit_path = generic_workflow_config[".bps_defined.submit_path"]
195 _LOG.info("Creating specific implementation of workflow")
196 wms_workflow = prepare(generic_workflow_config, generic_workflow, submit_path)
197 wms_workflow_config = generic_workflow_config
198 _LOG.info("Creating specific implementation of workflow took %.2f seconds", time.time() - stime)
199 print(f"Submit dir: {wms_workflow.submit_path}")
200 return wms_workflow_config, wms_workflow
203def submit_driver(config_file):
204 """Submit workflow for execution.
206 Parameters
207 ----------
208 config_file : `str`
209 Name of the configuration file.
210 """
211 wms_workflow_config, wms_workflow = prepare_driver(config_file)
212 submit(wms_workflow_config, wms_workflow)
213 print(f"Run Id: {wms_workflow.run_id}")
216def report_driver(wms_service, run_id, user, hist_days, pass_thru):
217 """Print out summary of jobs submitted for execution.
219 Parameters
220 ----------
221 wms_service : `str`
222 Name of the class.
223 run_id : `str`
224 A run id the report will be restricted to.
225 user : `str`
226 A user name the report will be restricted to.
227 hist_days : int
228 Number of days
229 pass_thru : `str`
230 A string to pass directly to the WMS service class.
231 """
232 report(wms_service, run_id, user, hist_days, pass_thru)
235def cancel_driver(wms_service, run_id, user, require_bps, pass_thru):
236 """Cancel submitted workflows.
238 Parameters
239 ----------
240 wms_service : `str`
241 Name of the Workload Management System service class.
242 run_id : `str`
243 ID or path of job that should be canceled.
244 user : `str`
245 User whose submitted jobs should be canceled.
246 require_bps : `bool`
247 Whether to require given run_id/user to be a bps submitted job.
248 pass_thru : `str`
249 Information to pass through to WMS.
250 """
251 cancel(wms_service, run_id, user, require_bps, pass_thru)