Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Driver to execute steps outside of BPS that need to be done first 

23including running QuantumGraph generation and reading the QuantumGraph 

24into memory. 

25""" 

26 

27import logging 

28import subprocess 

29import os 

30import shlex 

31import shutil 

32import time 

33 

34from lsst.daf.butler import DimensionUniverse 

35from lsst.pipe.base.graph import QuantumGraph 

36from lsst.utils import doImport 

37 

38_LOG = logging.getLogger() 

39 

40 

41def pre_transform(config, out_prefix=None): 

42 """Steps outside of BPS that need to be done first including generating 

43 a QuantumGraph. 

44 

45 Parameters 

46 ---------- 

47 config : `.bps_config.BpsConfig` 

48 Configuration values for BPS. In particular, looking for qgraph_file. 

49 out_prefix : `str` or None 

50 Output path for the QuantumGraph and stdout/stderr from generating 

51 the QuantumGraph. 

52 

53 Returns 

54 ------- 

55 qgraph_filename : `str` 

56 Name of file containing QuantumGraph that was read into qgraph. 

57 qgraph : `~lsst.pipe.base.graph.QuantumGraph` 

58 A QuantumGraph read in from pre-generated file or one that is the 

59 result of running code that generates it. 

60 """ 

61 # Check to see if user provided pre-generated QuantumGraph. 

62 found, input_qgraph_filename = config.search("qgraph_file") 

63 if found: 

64 if out_prefix is not None: 

65 # Save a copy of the QuantumGraph file in out_prefix. 

66 _LOG.info("Copying quantum graph (%s)", input_qgraph_filename) 

67 stime = time.time() 

68 qgraph_filename = os.path.join(out_prefix, os.path.basename(input_qgraph_filename)) 

69 shutil.copy2(input_qgraph_filename, qgraph_filename) 

70 _LOG.info("Copying quantum graph took %.2f seconds", time.time() - stime) 

71 else: 

72 # Use QuantumGraph file in original given location. 

73 qgraph_filename = input_qgraph_filename 

74 else: 

75 # Run command to create the QuantumGraph. 

76 _LOG.info("Creating quantum graph") 

77 stime = time.time() 

78 qgraph_filename = create_quantum_graph(config, out_prefix) 

79 _LOG.info("Creating quantum graph took %.2f seconds", time.time() - stime) 

80 

81 _LOG.info("Reading quantum graph (%s)", qgraph_filename) 

82 stime = time.time() 

83 qgraph = read_quantum_graph(qgraph_filename) 

84 _LOG.info("Reading quantum graph with %d nodes took %.2f seconds", len(qgraph), 

85 time.time() - stime) 

86 

87 return qgraph_filename, qgraph 

88 

89 

90def execute(command, filename): 

91 """Execute a command. 

92 

93 Parameters 

94 ---------- 

95 command : `str` 

96 String representing the command to execute. 

97 filename : `str` 

98 A file to which both stderr and stdout will be written to. 

99 

100 Returns 

101 ------- 

102 exit_code : `int` 

103 The exit code the command being executed finished with. 

104 """ 

105 buffer_size = 5000 

106 with open(filename, "w") as fh: 

107 print(command, file=fh) 

108 print("\n", file=fh) # Note: want a blank line 

109 process = subprocess.Popen( 

110 shlex.split(command), shell=False, stdout=subprocess.PIPE, 

111 stderr=subprocess.STDOUT 

112 ) 

113 buffer = os.read(process.stdout.fileno(), buffer_size).decode() 

114 while process.poll is None or buffer: 

115 print(buffer, end='', file=fh) 

116 _LOG.info(buffer) 

117 buffer = os.read(process.stdout.fileno(), buffer_size).decode() 

118 process.stdout.close() 

119 process.wait() 

120 return process.returncode 

121 

122 

123def create_quantum_graph(config, out_prefix=None): 

124 """Create QuantumGraph from pipeline definition 

125 

126 Parameters 

127 ---------- 

128 config : `.bps_config.BpsConfig` 

129 BPS configuration. 

130 out_prefix : `str` or None 

131 Path in which to output QuantumGraph as well as the stdout/stderr 

132 from generating the QuantumGraph. If out_prefix is None, code will write 

133 the QuantumGraph and stdout/stderr to the current directory. 

134 

135 Returns 

136 ------- 

137 qgraph_filename : `str` 

138 Name of file containing generated QuantumGraph 

139 """ 

140 # Create name of file to store QuantumGraph. 

141 qgraph_filename = f"{config['uniqProcName']}.pickle" 

142 if out_prefix is not None: 

143 qgraph_filename = os.path.join(out_prefix, qgraph_filename) 

144 

145 # Get QuantumGraph generation command. 

146 search_opt = {"curvals": {"qgraphFile": qgraph_filename}} 

147 found, cmd = config.search("createQuantumGraph", opt=search_opt) 

148 if not found: 

149 _LOG.error("command for generating QuantumGraph not found") 

150 _LOG.info(cmd) 

151 

152 # Run QuantumGraph generation. 

153 out = "quantumGraphGeneration.out" 

154 if out_prefix is not None: 

155 out = os.path.join(out_prefix, out) 

156 status = execute(cmd, out) 

157 if status != 0: 

158 raise RuntimeError(f"QuantumGraph generation exited with non-zero exit code ({status})\n" 

159 f"Check {out} for more details.") 

160 return qgraph_filename 

161 

162 

163def read_quantum_graph(qgraph_filename): 

164 """Read the QuantumGraph from disk. 

165 

166 Parameters 

167 ---------- 

168 qgraph_filename : `str` 

169 Name of file containing QuantumGraph to be used for workflow generation. 

170 

171 Returns 

172 ------- 

173 qgraph : `~lsst.pipe.base.graph.QuantumGraph` 

174 The QuantumGraph read from a file. 

175 

176 Raises 

177 ------ 

178 `RuntimeError` 

179 If the QuantumGraph contains 0 Quanta 

180 """ 

181 with open(qgraph_filename, "rb") as fh: 

182 qgraph = QuantumGraph.load(fh, DimensionUniverse()) 

183 if len(qgraph) == 0: 

184 raise RuntimeError("QuantumGraph is empty") 

185 return qgraph 

186 

187 

188def cluster_quanta(config, qgraph, name): 

189 """Call specified function to group quanta into clusters to be run together. 

190 

191 Parameters 

192 ---------- 

193 config : `~lsst.ctrl.bps.bps_config.BpsConfig` 

194 BPS configuration. 

195 qgraph : `~lsst.pipe.base.QuantumGraph` 

196 Original full QuantumGraph for the run. 

197 name : `str` 

198 Name for the ClusteredQuantumGraph that will be generated. 

199 

200 Returns 

201 ------- 

202 graph : `~lsst.ctrl.bps.clustered_quantum_graph.ClusteredQuantumGraph` 

203 Generated ClusteredQuantumGraph. 

204 """ 

205 cluster_func = doImport(config["clusterAlgorithm"]) 

206 return cluster_func(config, qgraph, name)