Coverage for python/lsst/ctrl/bps/panda/cmd_line_embedder.py: 24%
28 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-12 10:21 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-12 10:21 +0000
1# This file is part of ctrl_bps_panda.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import logging
23import os
25_LOG = logging.getLogger(__name__)
28class CommandLineEmbedder:
29 """
30 Class embeds static (constant across a task) values
31 into the pipeline execution command line
32 and resolves submission side environment variables
34 Parameters
35 ----------
36 config : `lsst.ctrl.bps.BpsConfig`
37 BPS configuration that includes the list of dynamic
38 (uniques per job) and submission side resolved variables
40 """
42 def __init__(self, config):
43 self.leave_placeholder_params = config.get("placeholderParams", ["qgraphNodeId", "qgraphId"])
44 self.submit_side_resolved = config.get("submitSideResolvedParams", ["USER"])
46 def replace_static_parameters(self, cmd_line, lazy_vars):
47 """Substitutes the lazy parameters in the command line which
48 are static, the same for every job in the workflow and could be
49 defined once. This function offloads the edge node processing
50 and number of parameters transferred together with job
53 Parameters
54 ----------
55 cmd_line: `str` command line to be processed
56 lazy_vars: `dict` of lazy variables and its values
58 Returns
59 -------
60 processed command line
61 """
62 for param_name, param_val in lazy_vars.items():
63 if param_name not in self.leave_placeholder_params:
64 cmd_line = cmd_line.replace("{" + param_name + "}", param_val)
65 return cmd_line
67 def resolve_submission_side_env_vars(self, cmd_line):
68 """Substitutes the lazy parameters in the command line
69 which are defined and resolved on the submission side
71 Parameters
72 ----------
73 cmd_line: `str` command line to be processed
75 Returns
76 -------
77 processed command line
78 """
80 for param in self.submit_side_resolved:
81 if os.getenv(param):
82 cmd_line = cmd_line.replace("<ENV:" + param + ">", os.getenv(param))
83 else:
84 _LOG.info("Expected parameter {0} is not found in the environment variables".format(param))
85 return cmd_line
87 def attach_pseudo_file_params(self, lazy_vars):
88 """Adds the parameters needed to finalize creation of a pseudo file
90 Parameters
91 ----------
92 lazy_vars: `dict` of values of to be substituted
94 Returns
95 -------
96 pseudo input file name suffix
97 """
99 file_suffix = ""
100 for item in self.leave_placeholder_params:
101 file_suffix += "+" + item + ":" + lazy_vars.get(item, "")
102 return file_suffix
104 def substitute_command_line(self, cmd_line, lazy_vars, job_name):
105 """Preprocesses the command line leaving for the egde node evaluation
106 only parameters which are job / environment dependent
108 Parameters
109 ----------
110 bps_file_name: `str` input file name proposed by BPS
111 cmd_line: `str` command line containing all lazy placeholders
112 lazy_vars: `dict` of lazy parameter name/values
113 job_name: `str` job name proposed by BPS
115 Returns
116 -------
117 cmd_line: `str`
118 processed command line
119 file_name: `str`
120 job pseudo input file name
121 """
123 cmd_line = self.replace_static_parameters(cmd_line, lazy_vars)
124 cmd_line = self.resolve_submission_side_env_vars(cmd_line)
125 file_name = job_name + self.attach_pseudo_file_params(lazy_vars)
126 return cmd_line, file_name