Coverage for python/lsst/ctrl/execute/configurator.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2#
3# LSST Data Management System
4# Copyright 2008-2016 LSST Corporation.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <http://www.lsstcorp.org/LegalNotices/>.
22#
24import os
25import os.path
26import pwd
27import sys
29import lsst.utils
30import eups
31from datetime import datetime
32from string import Template
33from .templateWriter import TemplateWriter
34from .condorConfig import CondorConfig
35from .condorInfoConfig import CondorInfoConfig
36from lsst.ctrl.execute import envString
39class Configurator:
40 """A class which consolidates Condor pex_config information with override
41 information (obtained from the command line) and produces Condor files
42 using these values.
43 """
45 def __init__(self, opts, configFileName):
46 """Constructor
47 @param opts: options to override
48 """
49 self.opts = opts
50 self.setup_using = None
51 self.manager = None
53 self.defaults = {}
55 fileName = envString.resolve(configFileName)
57 condorInfoConfig = CondorInfoConfig()
58 condorInfoConfig.load(fileName)
60 self.platform = self.opts.platform
62 # Look up the user's name and home directory in the
63 # $HOME//.lsst/condor-info.py file. If the platform
64 # is lsst, and the user_name or user_home is not in
65 # there, then default to user running this comman
66 # and the value of $HOME, respectively.
67 user_name = None
68 user_home = None
69 for name in list(condorInfoConfig.platform.keys()):
70 if name == self.platform:
71 user_name = condorInfoConfig.platform[name].user.name
72 user_home = condorInfoConfig.platform[name].user.home
74 # If we're on the lsst platform and the condorInfoConfig didn't
75 # have an entry for lsst user name and home, set to reasonable values
76 # These really do need to be set for all the other platforms, since
77 # while the user name may be the same, it's unlikely the home
78 # directory will be.
79 if self.platform == "lsst":
80 if user_name is None:
81 user_name = pwd.getpwuid(os.geteuid()).pw_name
82 if user_home is None:
83 user_home = os.getenv('HOME')
85 if user_name is None:
86 raise RuntimeError("error: %s does not specify user name for platform %s" %
87 (configFileName, self.platform))
88 if user_home is None:
89 raise RuntimeError("error: %s does not specify user home for platform %s" %
90 (configFileName, self.platform))
92 self.commandLineDefaults = {}
93 self.commandLineDefaults["USER_NAME"] = user_name
94 self.commandLineDefaults["USER_HOME"] = user_home
96 self.commandLineDefaults["DEFAULT_ROOT"] = self.opts.defaultRoot
97 self.commandLineDefaults["LOCAL_SCRATCH"] = self.opts.localScratch
98 self.commandLineDefaults["DATA_DIRECTORY"] = self.opts.dataDirectory
99 self.commandLineDefaults["IDS_PER_JOB"] = self.opts.idsPerJob
100 if self.opts.nodeSet is None:
101 self.commandLineDefaults["NODE_SET"] = ""
102 else:
103 self.commandLineDefaults["NODE_SET"] = self.opts.nodeSet
104 if self.opts.inputDataFile is None:
105 self.commandLineDefaults["INPUT_DATA_FILE"] = None
106 else:
107 self.commandLineDefaults["INPUT_DATA_FILE"] = os.path.abspath(self.opts.inputDataFile)
108 self.commandLineDefaults["FILE_SYSTEM_DOMAIN"] = self.opts.fileSystemDomain
109 self.commandLineDefaults["EUPS_PATH"] = self.opts.eupsPath
111 # override user name, if given
112 if self.opts.user_name is not None:
113 self.commandLineDefaults["USER_NAME"] = self.opts.user_name
115 # override user home, if given
116 if self.opts.user_home is not None:
117 self.commandLineDefaults["USER_HOME"] = self.opts.user_home
119 if self.opts.runid is not None:
120 self.runid = self.opts.runid
121 else:
122 self.runid = self.createRunId()
124 if self.opts.dagscript is not None:
125 self.commandLineDefaults["DAGSCRIPT"] = self.opts.dagscript
127 if self.opts.inputscript is not None:
128 self.commandLineDefaults["INPUTSCRIPT"] = self.opts.inputscript
130 if self.opts.platformConfig is not None:
131 self.commandLineDefaults["PLATFORM_CONFIG"] = self.opts.platformConfig
133 self.commandLineDefaults["COMMAND"] = self.opts.command
134 if self.commandLineDefaults["INPUT_DATA_FILE"] is not None:
135 self.commandLineDefaults["COMMAND"] = self.commandLineDefaults["COMMAND"]+" ${id_option}"
137 def getGenericConfigFileName(self):
138 """Retrieve a ctrl_execute orca config template, depending
139 on which target environment jobs will be running on.
140 @return the name of the orca config template
141 """
142 executePkgDir = lsst.utils.getPackageDir('ctrl_execute')
144 name = "config_with_%s.py.template" % self.setup_using
145 genericConfigName = os.path.join(executePkgDir,
146 "etc", "templates", self.manager, name)
147 if os.path.exists(genericConfigName):
148 return genericConfigName
149 raise RuntimeError("File %s not found; check etc/templates." %
150 genericConfigName)
152 def createRunId(self):
153 """create a unique runid
154 @return runid
155 """
156 # runid is in the form of <login>_YYYY_MMDD_HHMMSS
157 now = datetime.now()
158 username = pwd.getpwuid(os.geteuid()).pw_name
159 runid = "%s_%02d_%02d%02d_%02d%02d%02d" % (username, now.year, now.month,
160 now.day, now.hour, now.minute, now.second)
161 self.runid = runid
162 return runid
164 def getSetupPackages(self):
165 """Create a string of all the currently setup LSST software packages,
166 excluding any locally setup packages (LOCAL:). Also include any
167 packages specified on the comand line. This string will be
168 used to substitute within a preJob Template to create an LSST stack
169 environment that jobs will use.
170 @return string containing all setup commands, one per line.
171 """
172 e = eups.Eups()
173 setupProducts = e.getSetupProducts()
174 a = ""
176 # create a new list will all products and versions
177 allProducts = {}
178 for i in setupProducts:
179 allProducts[i.name] = i.version
181 # replace any existing products that we saw on the command line, adding
182 # them if they're not already there.
183 if self.opts.setup is not None:
184 for i, pkg in enumerate(self.opts.setup):
185 name = pkg[0]
186 version = pkg[1]
187 print("name = %s, version = %s" % (name, version))
188 allProducts[name] = version
190 # write out all products, except those that are setup locally.
191 for name in allProducts:
192 version = allProducts[name]
193 if self.platform == "lsst":
194 a = a + "setup -j %s %s\\n\\\n" % (name, version)
195 else:
196 if not version.startswith("LOCAL:"):
197 a = a + "setup -j %s %s\\n\\\n" % (name, version)
198 return a
200 def load(self, name):
201 """Loads all values from configuration and command line overrides into
202 data structures suitable for use by the TemplateWriter object.
203 """
204 resolvedName = envString.resolve(name)
205 configuration = CondorConfig()
206 configuration.load(resolvedName)
207 self.defaults = {}
209 if configuration.platform.nodeSetRequired and self.opts.nodeSet is None:
210 print("error: nodeset parameter required by this platform")
211 sys.exit(10)
213 tempDefaultRoot = Template(configuration.platform.defaultRoot)
214 self.defaults["DEFAULT_ROOT"] = tempDefaultRoot.substitute(
215 USER_NAME=self.commandLineDefaults["USER_NAME"])
217 tempLocalScratch = Template(configuration.platform.localScratch)
218 self.defaults["LOCAL_SCRATCH"] = \
219 tempLocalScratch.substitute(USER_NAME=self.commandLineDefaults["USER_NAME"])
220 self.defaults["IDS_PER_JOB"] = configuration.platform.idsPerJob
221 self.defaults["DATA_DIRECTORY"] = envString.resolve(configuration.platform.dataDirectory)
222 self.defaults["FILE_SYSTEM_DOMAIN"] = configuration.platform.fileSystemDomain
223 self.defaults["EUPS_PATH"] = configuration.platform.eupsPath
224 self.defaults["MANAGER_SOFTWARE_HOME"] = configuration.platform.manager_software_home
226 platform_dir = lsst.utils.getPackageDir("ctrl_platform_"+self.opts.platform)
227 self.defaults["PLATFORM_DIR"] = platform_dir
228 self.manager = configuration.platform.manager
229 self.setup_using = configuration.platform.setup_using
231 def createConfiguration(self, input):
232 """ creates a new Orca configuration file
233 @param input: template to use for value substitution
234 @return the newly created Orca configuration file
235 """
236 resolvedInputName = envString.resolve(input)
237 if self.opts.verbose:
238 print("creating configuration using ", resolvedInputName)
239 template = TemplateWriter()
240 substitutes = self.defaults.copy()
241 for key in self.commandLineDefaults:
242 val = self.commandLineDefaults[key]
243 if val is not None:
244 substitutes[key] = self.commandLineDefaults[key]
246 substitutes["CTRL_EXECUTE_SETUP_PACKAGES"] = self.getSetupPackages()
248 configDir = os.path.join(substitutes["LOCAL_SCRATCH"], "configs")
249 if not os.path.exists(configDir):
250 os.mkdir(configDir)
251 self.outputFileName = os.path.join(configDir, "%s.config" % (self.runid))
252 if self.opts.verbose:
253 print("writing new configuration to ", self.outputFileName)
254 template.rewrite(resolvedInputName, self.outputFileName, substitutes)
255 return self.outputFileName
257 def isVerbose(self):
258 """Checks to see if verbose flag was set.
259 @return value of verbose flag if it was set on the command line
260 """
261 return self.opts.verbose
263 def getParameter(self, value):
264 """Accessor for generic value
265 @return None if value is not set. Otherwise, use the comand line
266 override (if set), or the default Config value
267 """
268 if value in self.commandLineDefaults:
269 return self.commandLineDefaults[value]
270 if value in self.defaults:
271 return self.defaults[value]
272 return None
274 def getRunId(self):
275 """Accessor for run id
276 @return the value of the run id
277 """
278 return self.runid