Coverage for python/lsst/ctrl/bps/panda/cmd_line_embedder.py: 21%

33 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-01 02:18 -0800

1# This file is part of ctrl_bps_panda. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24import re 

25 

26_LOG = logging.getLogger(__name__) 

27 

28 

29class CommandLineEmbedder: 

30 """ 

31 Class embeds static (constant across a task) values 

32 into the pipeline execution command line 

33 and resolves submission side environment variables 

34 

35 Parameters 

36 ---------- 

37 config : `lsst.ctrl.bps.BpsConfig` 

38 BPS configuration that includes the list of dynamic 

39 (uniques per job) and submission side resolved variables 

40 

41 """ 

42 

43 def __init__(self, config): 

44 self.leave_placeholder_params = config.get("placeholderParams", ["qgraphNodeId", "qgraphId"]) 

45 self.submit_side_resolved = config.get("submitSideResolvedParams", ["USER"]) 

46 

47 def replace_static_parameters(self, cmd_line, lazy_vars): 

48 """Substitutes the lazy parameters in the command line which 

49 are static, the same for every job in the workflow and could be 

50 defined once. This function offloads the edge node processing 

51 and number of parameters transferred together with job 

52 

53 

54 Parameters 

55 ---------- 

56 cmd_line: `str` command line to be processed 

57 lazy_vars: `dict` of lazy variables and its values 

58 

59 Returns 

60 ------- 

61 processed command line 

62 """ 

63 for param_name, param_val in lazy_vars.items(): 

64 if param_name not in self.leave_placeholder_params: 

65 cmd_line = cmd_line.replace("{" + param_name + "}", param_val) 

66 return cmd_line 

67 

68 def resolve_submission_side_env_vars(self, cmd_line): 

69 """Substitutes the lazy parameters in the command line 

70 which are defined and resolved on the submission side 

71 

72 Parameters 

73 ---------- 

74 cmd_line: `str` command line to be processed 

75 

76 Returns 

77 ------- 

78 processed command line 

79 """ 

80 

81 for param in self.submit_side_resolved: 

82 if os.getenv(param): 

83 cmd_line = cmd_line.replace("<ENV:" + param + ">", os.getenv(param)) 

84 else: 

85 _LOG.info("Expected parameter {0} is not found in the environment variables".format(param)) 

86 return cmd_line 

87 

88 def attach_pseudo_file_params(self, lazy_vars): 

89 """Adds the parameters needed to finalize creation of a pseudo file 

90 

91 Parameters 

92 ---------- 

93 lazy_vars: `dict` of values of to be substituted 

94 

95 Returns 

96 ------- 

97 pseudo input file name suffix 

98 """ 

99 

100 file_suffix = "" 

101 for item in self.leave_placeholder_params: 

102 file_suffix += "+" + item + ":" + lazy_vars.get(item, "") 

103 return file_suffix 

104 

105 def substitute_command_line(self, cmd_line, lazy_vars, job_name): 

106 """Preprocesses the command line leaving for the egde node evaluation 

107 only parameters which are job / environment dependent 

108 

109 Parameters 

110 ---------- 

111 bps_file_name: `str` input file name proposed by BPS 

112 cmd_line: `str` command line containing all lazy placeholders 

113 lazy_vars: `dict` of lazy parameter name/values 

114 job_name: `str` job name proposed by BPS 

115 

116 Returns 

117 ------- 

118 cmd_line: `str` 

119 processed command line 

120 file_name: `str` 

121 job pseudo input file name 

122 """ 

123 cmd_vals = set([m.group(1) for m in re.finditer(r"[^$]{([^}]+)}", cmd_line)]) 

124 actual_lazy_vars = {} 

125 for key in cmd_vals: 

126 actual_lazy_vars[key] = lazy_vars[key] 

127 

128 cmd_line = self.replace_static_parameters(cmd_line, actual_lazy_vars) 

129 cmd_line = self.resolve_submission_side_env_vars(cmd_line) 

130 file_name = job_name + self.attach_pseudo_file_params(actual_lazy_vars) 

131 return cmd_line, file_name