Coverage for python/lsst/ctrl/bps/submit.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Driver for submitting a prepared WMS-specific workflow
23"""
25import getpass
26import logging
27import os
28import pickle
29import time
31from lsst.utils import doImport
32from lsst.obs.base import Instrument
34from .bps_draw import draw_networkx_dot
35from .pre_transform import pre_transform, cluster_quanta
36from .prepare import prepare
37from .transform import transform
40# Config section search order
41BPS_SEARCH_ORDER = ["payload", "pipetask", "site", "bps_defined"]
43# logging properties
44_LOG_PROP = """\
45log4j.rootLogger=INFO, A1
46log4j.appender.A1=ConsoleAppender
47log4j.appender.A1.Target=System.err
48log4j.appender.A1.layout=PatternLayout
49log4j.appender.A1.layout.ConversionPattern={}
50"""
52_LOG = logging.getLogger()
55def submit(config, wms_workflow, wms_service=None):
56 """Convert generic workflow to a workflow for a particular WMS.
58 Parameters
59 ----------
60 config : `~lsst.ctrl.bps.bps_config.BpsConfig`
61 Configuration values to be used by submission.
62 wms_workflow : `~lsst.ctrl.bps.wms_workflow.BaseWmsWorkflow`
63 The workflow to submit.
64 wms_service : `~lsst.ctrl.bps.wms_service.BaseWmsService`, optional
65 The workflow management service to which the workflow should be submitted.
67 Returns
68 -------
69 wms_workflow : `~lsst.ctrl.bps.wms_workflow.BaseWmsWorkflow`
70 WMS-specific workflow
71 """
72 if wms_service is None:
73 wms_service_class = doImport(config["wmsServiceClass"])
74 wms_service = wms_service_class(config)
75 return wms_service.submit(wms_workflow)
78def create_submission(config):
79 """Create submission files but don't actually submit.
81 Parameters
82 ----------
83 config : `~lsst.ctrl.bps.bps_config.BpsConfig`
84 Batch Processing service configuration.
86 Returns
87 -------
88 wms_workflow : `~lsst.ctrl.bps.wms_workflow`
89 WMS-specific workflow.
90 """
91 subtime = time.time()
93 config[".bps_defined.timestamp"] = Instrument.makeCollectionTimestamp()
94 if "uniqProcName" not in config:
95 config[".bps_defined.uniqProcName"] = config["outCollection"].replace("/", "_")
96 if "operator" not in config:
97 config[".bps_defined.operator"] = getpass.getuser()
99 # make submit directory to contain all outputs
100 submit_path = config["submitPath"]
101 os.makedirs(submit_path, exist_ok=True)
102 config[".bps_defined.submit_path"] = submit_path
104 stime = time.time()
105 _LOG.info("Pre-transform steps (includes QuantumGraph generation if needed)")
106 qgraph_file, qgraph = pre_transform(config, out_prefix=submit_path)
107 _LOG.info("Run QuantumGraph file %s", qgraph_file)
108 config['.bps_defined.run_qgraph_file'] = qgraph_file
109 clustered_qgraph = cluster_quanta(config, qgraph, config["uniqProcName"])
110 _LOG.info("Pre-transform steps took %.2f seconds", time.time() - stime)
111 with open(os.path.join(submit_path, "bps_clustered_qgraph.pickle"), 'wb') as outfh:
112 pickle.dump(clustered_qgraph, outfh)
113 _, save_dot = config.search("saveDot", opt={"default": False})
114 if save_dot:
115 draw_networkx_dot(clustered_qgraph, os.path.join(submit_path, "bps_clustered_qgraph.dot"))
117 stime = time.time()
118 _LOG.info("Creating Generic Workflow")
119 generic_workflow, generic_workflow_config = transform(config, clustered_qgraph, submit_path)
120 _LOG.info("Creating Generic Workflow took %.2f seconds", time.time() - stime)
121 _LOG.info("Generic Workflow name %s", generic_workflow.name)
122 with open(os.path.join(submit_path, "bps_generic_workflow.pickle"), 'wb') as outfh:
123 generic_workflow.save(outfh, 'pickle')
124 if save_dot:
125 with open(os.path.join(submit_path, "bps_generic_workflow.dot"), 'wb') as outfh:
126 generic_workflow.draw(outfh, 'dot')
128 stime = time.time()
129 _LOG.info("Creating specific implementation of workflow")
130 wms_workflow = prepare(generic_workflow_config, generic_workflow, submit_path)
131 _LOG.info("Creating specific implementation of workflow took %.2f seconds", time.time() - stime)
133 _LOG.info("Total submission creation time = %.2f", time.time() - subtime)
134 return wms_workflow