Coverage for python/lsst/ctrl/execute/configurator.py: 10%

151 statements  

« prev     ^ index     » next       coverage.py v7.2.1, created at 2023-03-12 01:44 -0800

1#!/usr/bin/env python 

2# 

3# LSST Data Management System 

4# Copyright 2008-2016 LSST Corporation. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <http://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24import os 

25import os.path 

26import pwd 

27import sys 

28 

29import lsst.utils 

30import eups 

31from datetime import datetime 

32from string import Template 

33from .templateWriter import TemplateWriter 

34from .condorConfig import CondorConfig 

35from .condorInfoConfig import CondorInfoConfig 

36from lsst.ctrl.execute import envString 

37 

38 

39class Configurator: 

40 """A class which consolidates Condor pex_config information with override 

41 information (obtained from the command line) and produces Condor files 

42 using these values. 

43 """ 

44 

45 def __init__(self, opts, configFileName): 

46 """Constructor 

47 @param opts: options to override 

48 """ 

49 self.opts = opts 

50 self.setup_using = None 

51 self.manager = None 

52 

53 self.defaults = {} 

54 

55 fileName = envString.resolve(configFileName) 

56 

57 condorInfoConfig = CondorInfoConfig() 

58 condorInfoConfig.load(fileName) 

59 

60 self.platform = self.opts.platform 

61 

62 # Look up the user's name and home directory in the 

63 # $HOME//.lsst/condor-info.py file. If the platform 

64 # is lsst, and the user_name or user_home is not in 

65 # there, then default to user running this comman 

66 # and the value of $HOME, respectively. 

67 user_name = None 

68 user_home = None 

69 for name in list(condorInfoConfig.platform.keys()): 

70 if name == self.platform: 

71 user_name = condorInfoConfig.platform[name].user.name 

72 user_home = condorInfoConfig.platform[name].user.home 

73 

74 # If we're on the lsst platform and the condorInfoConfig didn't 

75 # have an entry for lsst user name and home, set to reasonable values 

76 # These really do need to be set for all the other platforms, since 

77 # while the user name may be the same, it's unlikely the home 

78 # directory will be. 

79 if self.platform == "lsst": 

80 if user_name is None: 

81 user_name = pwd.getpwuid(os.geteuid()).pw_name 

82 if user_home is None: 

83 user_home = os.getenv('HOME') 

84 

85 if user_name is None: 

86 raise RuntimeError("error: %s does not specify user name for platform %s" % 

87 (configFileName, self.platform)) 

88 if user_home is None: 

89 raise RuntimeError("error: %s does not specify user home for platform %s" % 

90 (configFileName, self.platform)) 

91 

92 self.commandLineDefaults = {} 

93 self.commandLineDefaults["USER_NAME"] = user_name 

94 self.commandLineDefaults["USER_HOME"] = user_home 

95 

96 self.commandLineDefaults["DEFAULT_ROOT"] = self.opts.defaultRoot 

97 self.commandLineDefaults["LOCAL_SCRATCH"] = self.opts.localScratch 

98 self.commandLineDefaults["DATA_DIRECTORY"] = self.opts.dataDirectory 

99 self.commandLineDefaults["IDS_PER_JOB"] = self.opts.idsPerJob 

100 if self.opts.nodeSet is None: 

101 self.commandLineDefaults["NODE_SET"] = "" 

102 else: 

103 self.commandLineDefaults["NODE_SET"] = self.opts.nodeSet 

104 if self.opts.inputDataFile is None: 

105 self.commandLineDefaults["INPUT_DATA_FILE"] = None 

106 else: 

107 self.commandLineDefaults["INPUT_DATA_FILE"] = os.path.abspath(self.opts.inputDataFile) 

108 self.commandLineDefaults["FILE_SYSTEM_DOMAIN"] = self.opts.fileSystemDomain 

109 self.commandLineDefaults["EUPS_PATH"] = self.opts.eupsPath 

110 

111 # override user name, if given 

112 if self.opts.user_name is not None: 

113 self.commandLineDefaults["USER_NAME"] = self.opts.user_name 

114 

115 # override user home, if given 

116 if self.opts.user_home is not None: 

117 self.commandLineDefaults["USER_HOME"] = self.opts.user_home 

118 

119 if self.opts.runid is not None: 

120 self.runid = self.opts.runid 

121 else: 

122 self.runid = self.createRunId() 

123 

124 if self.opts.dagscript is not None: 

125 self.commandLineDefaults["DAGSCRIPT"] = self.opts.dagscript 

126 

127 if self.opts.inputscript is not None: 

128 self.commandLineDefaults["INPUTSCRIPT"] = self.opts.inputscript 

129 

130 if self.opts.platformConfig is not None: 

131 self.commandLineDefaults["PLATFORM_CONFIG"] = self.opts.platformConfig 

132 

133 self.commandLineDefaults["COMMAND"] = self.opts.command 

134 if self.commandLineDefaults["INPUT_DATA_FILE"] is not None: 

135 self.commandLineDefaults["COMMAND"] = self.commandLineDefaults["COMMAND"]+" ${id_option}" 

136 

137 def getGenericConfigFileName(self): 

138 """Retrieve a ctrl_execute orca config template, depending 

139 on which target environment jobs will be running on. 

140 @return the name of the orca config template 

141 """ 

142 executePkgDir = lsst.utils.getPackageDir('ctrl_execute') 

143 

144 name = "config_with_%s.py.template" % self.setup_using 

145 genericConfigName = os.path.join(executePkgDir, 

146 "etc", "templates", self.manager, name) 

147 if os.path.exists(genericConfigName): 

148 return genericConfigName 

149 raise RuntimeError("File %s not found; check etc/templates." % 

150 genericConfigName) 

151 

152 def createRunId(self): 

153 """create a unique runid 

154 @return runid 

155 """ 

156 # runid is in the form of <login>_YYYY_MMDD_HHMMSS 

157 now = datetime.now() 

158 username = pwd.getpwuid(os.geteuid()).pw_name 

159 runid = "%s_%02d_%02d%02d_%02d%02d%02d" % (username, now.year, now.month, 

160 now.day, now.hour, now.minute, now.second) 

161 self.runid = runid 

162 return runid 

163 

164 def getSetupPackages(self): 

165 """Create a string of all the currently setup LSST software packages, 

166 excluding any locally setup packages (LOCAL:). Also include any 

167 packages specified on the comand line. This string will be 

168 used to substitute within a preJob Template to create an LSST stack 

169 environment that jobs will use. 

170 @return string containing all setup commands, one per line. 

171 """ 

172 e = eups.Eups() 

173 setupProducts = e.getSetupProducts() 

174 a = "" 

175 

176 # create a new list will all products and versions 

177 allProducts = {} 

178 for i in setupProducts: 

179 allProducts[i.name] = i.version 

180 

181 # replace any existing products that we saw on the command line, adding 

182 # them if they're not already there. 

183 if self.opts.setup is not None: 

184 for i, pkg in enumerate(self.opts.setup): 

185 name = pkg[0] 

186 version = pkg[1] 

187 print("name = %s, version = %s" % (name, version)) 

188 allProducts[name] = version 

189 

190 # write out all products, except those that are setup locally. 

191 for name in allProducts: 

192 version = allProducts[name] 

193 if self.platform == "lsst": 

194 a = a + "setup -j %s %s\\n\\\n" % (name, version) 

195 else: 

196 if not version.startswith("LOCAL:"): 

197 a = a + "setup -j %s %s\\n\\\n" % (name, version) 

198 return a 

199 

200 def load(self, name): 

201 """Loads all values from configuration and command line overrides into 

202 data structures suitable for use by the TemplateWriter object. 

203 """ 

204 resolvedName = envString.resolve(name) 

205 configuration = CondorConfig() 

206 configuration.load(resolvedName) 

207 self.defaults = {} 

208 

209 if configuration.platform.nodeSetRequired and self.opts.nodeSet is None: 

210 print("error: nodeset parameter required by this platform") 

211 sys.exit(10) 

212 

213 tempDefaultRoot = Template(configuration.platform.defaultRoot) 

214 self.defaults["DEFAULT_ROOT"] = tempDefaultRoot.substitute( 

215 USER_NAME=self.commandLineDefaults["USER_NAME"]) 

216 

217 tempLocalScratch = Template(configuration.platform.localScratch) 

218 self.defaults["LOCAL_SCRATCH"] = \ 

219 tempLocalScratch.substitute(USER_NAME=self.commandLineDefaults["USER_NAME"]) 

220 self.defaults["IDS_PER_JOB"] = configuration.platform.idsPerJob 

221 self.defaults["DATA_DIRECTORY"] = envString.resolve(configuration.platform.dataDirectory) 

222 self.defaults["FILE_SYSTEM_DOMAIN"] = configuration.platform.fileSystemDomain 

223 self.defaults["EUPS_PATH"] = configuration.platform.eupsPath 

224 self.defaults["MANAGER_SOFTWARE_HOME"] = configuration.platform.manager_software_home 

225 

226 platform_dir = lsst.utils.getPackageDir("ctrl_platform_"+self.opts.platform) 

227 self.defaults["PLATFORM_DIR"] = platform_dir 

228 self.manager = configuration.platform.manager 

229 self.setup_using = configuration.platform.setup_using 

230 

231 def createConfiguration(self, input): 

232 """ creates a new Orca configuration file 

233 @param input: template to use for value substitution 

234 @return the newly created Orca configuration file 

235 """ 

236 resolvedInputName = envString.resolve(input) 

237 if self.opts.verbose: 

238 print("creating configuration using ", resolvedInputName) 

239 template = TemplateWriter() 

240 substitutes = self.defaults.copy() 

241 for key in self.commandLineDefaults: 

242 val = self.commandLineDefaults[key] 

243 if val is not None: 

244 substitutes[key] = self.commandLineDefaults[key] 

245 

246 substitutes["CTRL_EXECUTE_SETUP_PACKAGES"] = self.getSetupPackages() 

247 

248 configDir = os.path.join(substitutes["LOCAL_SCRATCH"], "configs") 

249 if not os.path.exists(configDir): 

250 os.mkdir(configDir) 

251 self.outputFileName = os.path.join(configDir, "%s.config" % (self.runid)) 

252 if self.opts.verbose: 

253 print("writing new configuration to ", self.outputFileName) 

254 template.rewrite(resolvedInputName, self.outputFileName, substitutes) 

255 return self.outputFileName 

256 

257 def isVerbose(self): 

258 """Checks to see if verbose flag was set. 

259 @return value of verbose flag if it was set on the command line 

260 """ 

261 return self.opts.verbose 

262 

263 def getParameter(self, value): 

264 """Accessor for generic value 

265 @return None if value is not set. Otherwise, use the comand line 

266 override (if set), or the default Config value 

267 """ 

268 if value in self.commandLineDefaults: 

269 return self.commandLineDefaults[value] 

270 if value in self.defaults: 

271 return self.defaults[value] 

272 return None 

273 

274 def getRunId(self): 

275 """Accessor for run id 

276 @return the value of the run id 

277 """ 

278 return self.runid