Coverage for python/lsst/ctrl/bps/panda/cmd_line_embedder.py: 21%

33 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-10 10:39 +0000

1# This file is part of ctrl_bps_panda. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24import re 

25 

26_LOG = logging.getLogger(__name__) 

27 

28 

29class CommandLineEmbedder: 

30 """Class embeds static (constant across a task) values 

31 into the pipeline execution command line 

32 and resolves submission side environment variables 

33 

34 Parameters 

35 ---------- 

36 config : `lsst.ctrl.bps.BpsConfig` 

37 BPS configuration that includes the list of dynamic 

38 (uniques per job) and submission side resolved variables 

39 """ 

40 

41 def __init__(self, config): 

42 self.leave_placeholder_params = config.get("placeholderParams", ["qgraphNodeId", "qgraphId"]) 

43 self.submit_side_resolved = config.get("submitSideResolvedParams", ["USER"]) 

44 

45 def replace_static_parameters(self, cmd_line, lazy_vars): 

46 """Substitutes the lazy parameters in the command line which 

47 are static, the same for every job in the workflow and could be 

48 defined once. 

49 

50 This function offloads the edge node processing 

51 and number of parameters transferred together with job 

52 

53 Parameters 

54 ---------- 

55 cmd_line: `str` 

56 Command line to be processed. 

57 lazy_vars : `dict` 

58 Lazy variables and its values. 

59 

60 Returns 

61 ------- 

62 cmd : `str` 

63 Processed command line. 

64 """ 

65 for param_name, param_val in lazy_vars.items(): 

66 if param_name not in self.leave_placeholder_params: 

67 cmd_line = cmd_line.replace("{" + param_name + "}", param_val) 

68 return cmd_line 

69 

70 def resolve_submission_side_env_vars(self, cmd_line): 

71 """Substitute the lazy parameters in the command line 

72 which are defined and resolved on the submission side. 

73 

74 Parameters 

75 ---------- 

76 cmd_line : `str` 

77 Command line to be processed. 

78 

79 Returns 

80 ------- 

81 cmd : `str` 

82 Processed command line. 

83 """ 

84 for param in self.submit_side_resolved: 

85 if os.getenv(param): 

86 cmd_line = cmd_line.replace("<ENV:" + param + ">", os.getenv(param)) 

87 else: 

88 _LOG.info("Expected parameter %s is not found in the environment variables", param) 

89 return cmd_line 

90 

91 def attach_pseudo_file_params(self, lazy_vars): 

92 """Add the parameters needed to finalize creation of a pseudo file. 

93 

94 Parameters 

95 ---------- 

96 lazy_vars : `dict` 

97 Values to be substituted. 

98 

99 Returns 

100 ------- 

101 suffix : `str` 

102 Pseudo input file name suffix. 

103 """ 

104 file_suffix = "" 

105 for item in self.leave_placeholder_params: 

106 file_suffix += "+" + item + ":" + lazy_vars.get(item, "") 

107 return file_suffix 

108 

109 def substitute_command_line(self, cmd_line, lazy_vars, job_name): 

110 """Preprocess the command line leaving for the egde node evaluation 

111 only parameters which are job / environment dependent. 

112 

113 Parameters 

114 ---------- 

115 bps_file_name : `str` 

116 Input file name proposed by BPS. 

117 cmd_line : `str` 

118 Command line containing all lazy placeholders. 

119 lazy_vars : `dict` 

120 Lazy parameter name/values. 

121 job_name : `str` 

122 Job name proposed by BPS. 

123 

124 Returns 

125 ------- 

126 cmd_line: `str` 

127 processed command line 

128 file_name: `str` 

129 job pseudo input file name 

130 """ 

131 cmd_vals = {m.group(1) for m in re.finditer(r"[^$]{([^}]+)}", cmd_line)} 

132 actual_lazy_vars = {} 

133 for key in cmd_vals: 

134 actual_lazy_vars[key] = lazy_vars[key] 

135 

136 cmd_line = self.replace_static_parameters(cmd_line, actual_lazy_vars) 

137 cmd_line = self.resolve_submission_side_env_vars(cmd_line) 

138 file_name = job_name + self.attach_pseudo_file_params(actual_lazy_vars) 

139 return cmd_line, file_name