Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

# This file is part of ctrl_mpexec. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (https://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <https://www.gnu.org/licenses/>. 

 

"""Module defining TaskLoader class and related methods. 

""" 

 

__all__ = ['TaskLoader'] 

 

# ------------------------------- 

# Imports of standard modules -- 

# ------------------------------- 

import importlib 

import inspect 

import logging 

import pkgutil 

 

# ----------------------------- 

# Imports for other modules -- 

# ----------------------------- 

from lsst.pipe.base import CmdLineTask, Task, PipelineTask 

 

# ---------------------------------- 

# Local non-exported definitions -- 

# ---------------------------------- 

 

_LOG = logging.getLogger(__name__.partition(".")[2]) 

 

 

def _task_kind(task_class): 

"""Determine task kind. 

 

Parameters 

---------- 

task_class 

Python class object 

 

Returns 

------- 

None if `task_class` is not a class or does not inherit from Task. 

Otherwise returns one of KIND_TASK, KIND_CMDLINETASK, or KIND_PIPELINETASK. 

""" 

kind = None 

if inspect.isclass(task_class): 

bases = inspect.getmro(task_class) 

if PipelineTask in bases: 

kind = KIND_PIPELINETASK 

elif CmdLineTask in bases: 

kind = KIND_CMDLINETASK 

elif Task in bases: 

kind = KIND_TASK 

return kind 

 

# ------------------------ 

# Exported definitions -- 

# ------------------------ 

 

 

KIND_TASK = 'Task' 

KIND_CMDLINETASK = 'CmdLineTask' 

KIND_PIPELINETASK = 'PipelineTask' 

 

 

class TaskLoader: 

"""Task responsible for finding and loading tasks. 

 

Parameters 

---------- 

packages : `list` of `str`, optional 

Defines the set of package names to look for tasks. There is a small 

pre-defined set of packages that is used by default. 

""" 

 

# default locations for packages 

# TODO: examples should be removed later. 

DEFAULT_PACKAGES = ['lsst.ctrl.mpexec.examples', 'lsst.pipe.tasks'] 

 

def __init__(self, packages=None): 

if not packages: 

packages = TaskLoader.DEFAULT_PACKAGES 

self._packages = packages 

 

@property 

def packages(self): 

"""Return current set of packages in search path. 

""" 

return self._packages 

 

def modules(self): 

"""Return set of modules and sub-packages found in the packages. 

 

Returns 

------- 

`list` of tuples (name, flag), `name` is the module or sub-package 

name (includes dot-separated parent package name), `flag` is set to 

False for module and True for sub-package. 

 

Raises 

------ 

`ImportError` 

If fails to import any package 

""" 

modules = [] 

for pkg_name in self._packages: 

_LOG.debug("get modules from package %r", pkg_name) 

pkg = importlib.import_module(pkg_name) 

for _, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, pkg.__name__ + '.'): 

_LOG.debug("found module %r ispkg=%s", module_name, ispkg) 

modules.append((module_name, ispkg)) 

return modules 

 

def tasks(self): 

"""Return list of all tasks in the packages. 

 

Returns 

------- 

`list` of tuples (name, kind), `name` is the full task name including package 

and module name, `kind` is a task kind, one of the constants `KIND_TASK`, 

`KIND_CMDLINETASK`, or `KIND_PIPELINETASK`. 

 

Raises 

------ 

`ImportError` 

If fails to import any package 

""" 

tasks = [] 

for pkg_name in self._packages: 

_LOG.debug("importing package %r", pkg_name) 

pkg = importlib.import_module(pkg_name) 

for _, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, pkg.__name__ + '.'): 

_LOG.debug("found module %r ispkg=%s", module_name, ispkg) 

# classes should not live in packages 

if not ispkg: 

try: 

_LOG.debug("importing module %r", module_name) 

mod = importlib.import_module(module_name) 

except ImportError as exc: 

_LOG.debug("import failed: %s", exc) 

else: 

for name, obj in vars(mod).items(): 

if inspect.isclass(obj) and inspect.getmodule(obj) is mod: 

kind = _task_kind(obj) 

if kind is not None: 

tasks.append((module_name + '.' + name, kind)) 

return tasks 

 

def loadTaskClass(self, task_class_name): 

"""Find and load a task class. 

 

If `task_class_name` is a simple identifier without dots then search 

for that class in all modules in package list, this means importing 

all modules which can be slow. 

 

Otherwise if `task_class_name` has dots then we try to import it 

directly assuming name is ``package.module.Class``, if that fails 

then try importing it assuming its name is relative to a package 

names in the known package list. 

 

Parameters 

---------- 

task_class_name : `str` 

Task class name which can include package and module names 

separated by dots. 

 

Returns 

------- 

task_class 

Python class object for a task, `None` if class was not found 

taks_name 

fully-qualified class name ("package.module.TaskClass") 

taks_kind 

one of KIND_TASK, KIND_CMDLINETASK, or KIND_PIPELINETASK 

 

Raises 

------ 

`ImportError` is raised when task class cannot be imported. 

""" 

_LOG.debug("load_task_class: will look for %r", task_class_name) 

dot = task_class_name.rfind('.') 

if dot > 0: 

 

# name is package.module.Class or module.Class, either absolute 

# or relative to package list 

module_name = task_class_name[:dot] 

class_name = task_class_name[dot + 1:] 

 

for package in [None] + self._packages: 

 

full_module_name = module_name 

if package: 

full_module_name = package + '.' + full_module_name 

 

try: 

_LOG.debug("load_task_class: try module %r", full_module_name) 

module = importlib.import_module(full_module_name) 

_LOG.debug("load_task_class: imported %r", full_module_name) 

except ImportError: 

pass 

else: 

# get Class from module, if not there try other options 

klass = getattr(module, class_name, None) 

if klass is not None: 

kind = _task_kind(klass) 

_LOG.debug("load_task_class: found %r in %r, kind: %s", 

class_name, full_module_name, kind) 

if kind is not None: 

return (klass, full_module_name + '.' + class_name, kind) 

else: 

_LOG.debug("load_task_class: no class %r in module %r", 

class_name, full_module_name) 

 

else: 

 

# simple name, search for it in all modules in every package, not 

# very efficient 

for pkg_name in self._packages: 

_LOG.debug("load_task_class: importing package %r", pkg_name) 

pkg = importlib.import_module(pkg_name) 

for _, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, pkg.__name__ + '.'): 

_LOG.debug("load_task_class: found module %r ispkg=%s", module_name, ispkg) 

# classes should not live in packages 

if not ispkg: 

try: 

_LOG.debug("load_task_class: importing module %r", module_name) 

mod = importlib.import_module(module_name) 

except ImportError as exc: 

_LOG.debug("load_task_class: import failed: %s", exc) 

else: 

obj = getattr(mod, task_class_name, None) 

if inspect.isclass(obj) and inspect.getmodule(obj) is mod: 

kind = _task_kind(obj) 

_LOG.debug("load_task_class: found class %r kind: %s", 

task_class_name, kind) 

if kind is not None: 

return (obj, module_name + '.' + task_class_name, kind) 

 

raise ImportError('Cannot find class named "' + task_class_name + '" in known packages')