Coverage for python/lsst/ctrl/bps/panda/cmd_line_embedder.py: 24%

28 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-06 02:12 -0800

1# This file is part of ctrl_bps_panda. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24 

25_LOG = logging.getLogger(__name__) 

26 

27 

28class CommandLineEmbedder: 

29 """ 

30 Class embeds static (constant across a task) values 

31 into the pipeline execution command line 

32 and resolves submission side environment variables 

33 

34 Parameters 

35 ---------- 

36 config : `lsst.ctrl.bps.BpsConfig` 

37 BPS configuration that includes the list of dynamic 

38 (uniques per job) and submission side resolved variables 

39 

40 """ 

41 

42 def __init__(self, config): 

43 self.leave_placeholder_params = config.get("placeholderParams", ["qgraphNodeId", "qgraphId"]) 

44 self.submit_side_resolved = config.get("submitSideResolvedParams", ["USER"]) 

45 

46 def replace_static_parameters(self, cmd_line, lazy_vars): 

47 """Substitutes the lazy parameters in the command line which 

48 are static, the same for every job in the workflow and could be 

49 defined once. This function offloads the edge node processing 

50 and number of parameters transferred together with job 

51 

52 

53 Parameters 

54 ---------- 

55 cmd_line: `str` command line to be processed 

56 lazy_vars: `dict` of lazy variables and its values 

57 

58 Returns 

59 ------- 

60 processed command line 

61 """ 

62 for param_name, param_val in lazy_vars.items(): 

63 if param_name not in self.leave_placeholder_params: 

64 cmd_line = cmd_line.replace("{" + param_name + "}", param_val) 

65 return cmd_line 

66 

67 def resolve_submission_side_env_vars(self, cmd_line): 

68 """Substitutes the lazy parameters in the command line 

69 which are defined and resolved on the submission side 

70 

71 Parameters 

72 ---------- 

73 cmd_line: `str` command line to be processed 

74 

75 Returns 

76 ------- 

77 processed command line 

78 """ 

79 

80 for param in self.submit_side_resolved: 

81 if os.getenv(param): 

82 cmd_line = cmd_line.replace("<ENV:" + param + ">", os.getenv(param)) 

83 else: 

84 _LOG.info("Expected parameter {0} is not found in the environment variables".format(param)) 

85 return cmd_line 

86 

87 def attach_pseudo_file_params(self, lazy_vars): 

88 """Adds the parameters needed to finalize creation of a pseudo file 

89 

90 Parameters 

91 ---------- 

92 lazy_vars: `dict` of values of to be substituted 

93 

94 Returns 

95 ------- 

96 pseudo input file name suffix 

97 """ 

98 

99 file_suffix = "" 

100 for item in self.leave_placeholder_params: 

101 file_suffix += "+" + item + ":" + lazy_vars.get(item, "") 

102 return file_suffix 

103 

104 def substitute_command_line(self, cmd_line, lazy_vars, job_name): 

105 """Preprocesses the command line leaving for the egde node evaluation 

106 only parameters which are job / environment dependent 

107 

108 Parameters 

109 ---------- 

110 bps_file_name: `str` input file name proposed by BPS 

111 cmd_line: `str` command line containing all lazy placeholders 

112 lazy_vars: `dict` of lazy parameter name/values 

113 job_name: `str` job name proposed by BPS 

114 

115 Returns 

116 ------- 

117 cmd_line: `str` 

118 processed command line 

119 file_name: `str` 

120 job pseudo input file name 

121 """ 

122 

123 cmd_line = self.replace_static_parameters(cmd_line, lazy_vars) 

124 cmd_line = self.resolve_submission_side_env_vars(cmd_line) 

125 file_name = job_name + self.attach_pseudo_file_params(lazy_vars) 

126 return cmd_line, file_name