Coverage for python/lsst/ctrl/bps/panda/cmd_line_embedder.py: 21%
33 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-09 09:28 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-09 09:28 +0000
1# This file is part of ctrl_bps_panda.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import logging
23import os
24import re
26_LOG = logging.getLogger(__name__)
29class CommandLineEmbedder:
30 """
31 Class embeds static (constant across a task) values
32 into the pipeline execution command line
33 and resolves submission side environment variables
35 Parameters
36 ----------
37 config : `lsst.ctrl.bps.BpsConfig`
38 BPS configuration that includes the list of dynamic
39 (uniques per job) and submission side resolved variables
41 """
43 def __init__(self, config):
44 self.leave_placeholder_params = config.get("placeholderParams", ["qgraphNodeId", "qgraphId"])
45 self.submit_side_resolved = config.get("submitSideResolvedParams", ["USER"])
47 def replace_static_parameters(self, cmd_line, lazy_vars):
48 """Substitutes the lazy parameters in the command line which
49 are static, the same for every job in the workflow and could be
50 defined once. This function offloads the edge node processing
51 and number of parameters transferred together with job
54 Parameters
55 ----------
56 cmd_line: `str` command line to be processed
57 lazy_vars: `dict` of lazy variables and its values
59 Returns
60 -------
61 processed command line
62 """
63 for param_name, param_val in lazy_vars.items():
64 if param_name not in self.leave_placeholder_params:
65 cmd_line = cmd_line.replace("{" + param_name + "}", param_val)
66 return cmd_line
68 def resolve_submission_side_env_vars(self, cmd_line):
69 """Substitutes the lazy parameters in the command line
70 which are defined and resolved on the submission side
72 Parameters
73 ----------
74 cmd_line: `str` command line to be processed
76 Returns
77 -------
78 processed command line
79 """
81 for param in self.submit_side_resolved:
82 if os.getenv(param):
83 cmd_line = cmd_line.replace("<ENV:" + param + ">", os.getenv(param))
84 else:
85 _LOG.info("Expected parameter {0} is not found in the environment variables".format(param))
86 return cmd_line
88 def attach_pseudo_file_params(self, lazy_vars):
89 """Adds the parameters needed to finalize creation of a pseudo file
91 Parameters
92 ----------
93 lazy_vars: `dict` of values of to be substituted
95 Returns
96 -------
97 pseudo input file name suffix
98 """
100 file_suffix = ""
101 for item in self.leave_placeholder_params:
102 file_suffix += "+" + item + ":" + lazy_vars.get(item, "")
103 return file_suffix
105 def substitute_command_line(self, cmd_line, lazy_vars, job_name):
106 """Preprocesses the command line leaving for the egde node evaluation
107 only parameters which are job / environment dependent
109 Parameters
110 ----------
111 bps_file_name: `str` input file name proposed by BPS
112 cmd_line: `str` command line containing all lazy placeholders
113 lazy_vars: `dict` of lazy parameter name/values
114 job_name: `str` job name proposed by BPS
116 Returns
117 -------
118 cmd_line: `str`
119 processed command line
120 file_name: `str`
121 job pseudo input file name
122 """
123 cmd_vals = set([m.group(1) for m in re.finditer(r"[^$]{([^}]+)}", cmd_line)])
124 actual_lazy_vars = {}
125 for key in cmd_vals:
126 actual_lazy_vars[key] = lazy_vars[key]
128 cmd_line = self.replace_static_parameters(cmd_line, actual_lazy_vars)
129 cmd_line = self.resolve_submission_side_env_vars(cmd_line)
130 file_name = job_name + self.attach_pseudo_file_params(actual_lazy_vars)
131 return cmd_line, file_name