Coverage for python/lsst/ctrl/bps/panda/cmd_line_embedder.py: 21%
33 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-10 10:39 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-10 10:39 +0000
1# This file is part of ctrl_bps_panda.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import logging
23import os
24import re
26_LOG = logging.getLogger(__name__)
29class CommandLineEmbedder:
30 """Class embeds static (constant across a task) values
31 into the pipeline execution command line
32 and resolves submission side environment variables
34 Parameters
35 ----------
36 config : `lsst.ctrl.bps.BpsConfig`
37 BPS configuration that includes the list of dynamic
38 (uniques per job) and submission side resolved variables
39 """
41 def __init__(self, config):
42 self.leave_placeholder_params = config.get("placeholderParams", ["qgraphNodeId", "qgraphId"])
43 self.submit_side_resolved = config.get("submitSideResolvedParams", ["USER"])
45 def replace_static_parameters(self, cmd_line, lazy_vars):
46 """Substitutes the lazy parameters in the command line which
47 are static, the same for every job in the workflow and could be
48 defined once.
50 This function offloads the edge node processing
51 and number of parameters transferred together with job
53 Parameters
54 ----------
55 cmd_line: `str`
56 Command line to be processed.
57 lazy_vars : `dict`
58 Lazy variables and its values.
60 Returns
61 -------
62 cmd : `str`
63 Processed command line.
64 """
65 for param_name, param_val in lazy_vars.items():
66 if param_name not in self.leave_placeholder_params:
67 cmd_line = cmd_line.replace("{" + param_name + "}", param_val)
68 return cmd_line
70 def resolve_submission_side_env_vars(self, cmd_line):
71 """Substitute the lazy parameters in the command line
72 which are defined and resolved on the submission side.
74 Parameters
75 ----------
76 cmd_line : `str`
77 Command line to be processed.
79 Returns
80 -------
81 cmd : `str`
82 Processed command line.
83 """
84 for param in self.submit_side_resolved:
85 if os.getenv(param):
86 cmd_line = cmd_line.replace("<ENV:" + param + ">", os.getenv(param))
87 else:
88 _LOG.info("Expected parameter %s is not found in the environment variables", param)
89 return cmd_line
91 def attach_pseudo_file_params(self, lazy_vars):
92 """Add the parameters needed to finalize creation of a pseudo file.
94 Parameters
95 ----------
96 lazy_vars : `dict`
97 Values to be substituted.
99 Returns
100 -------
101 suffix : `str`
102 Pseudo input file name suffix.
103 """
104 file_suffix = ""
105 for item in self.leave_placeholder_params:
106 file_suffix += "+" + item + ":" + lazy_vars.get(item, "")
107 return file_suffix
109 def substitute_command_line(self, cmd_line, lazy_vars, job_name):
110 """Preprocess the command line leaving for the egde node evaluation
111 only parameters which are job / environment dependent.
113 Parameters
114 ----------
115 bps_file_name : `str`
116 Input file name proposed by BPS.
117 cmd_line : `str`
118 Command line containing all lazy placeholders.
119 lazy_vars : `dict`
120 Lazy parameter name/values.
121 job_name : `str`
122 Job name proposed by BPS.
124 Returns
125 -------
126 cmd_line: `str`
127 processed command line
128 file_name: `str`
129 job pseudo input file name
130 """
131 cmd_vals = {m.group(1) for m in re.finditer(r"[^$]{([^}]+)}", cmd_line)}
132 actual_lazy_vars = {}
133 for key in cmd_vals:
134 actual_lazy_vars[key] = lazy_vars[key]
136 cmd_line = self.replace_static_parameters(cmd_line, actual_lazy_vars)
137 cmd_line = self.resolve_submission_side_env_vars(cmd_line)
138 file_name = job_name + self.attach_pseudo_file_params(actual_lazy_vars)
139 return cmd_line, file_name