Coverage for python/lsst/pipe/base/task.py: 31%

109 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-06-06 10:05 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23from __future__ import annotations 

24 

25__all__ = ["Task", "TaskError"] 

26 

27import contextlib 

28import logging 

29import weakref 

30from typing import ( 

31 TYPE_CHECKING, 

32 Any, 

33 Callable, 

34 ClassVar, 

35 Dict, 

36 Iterator, 

37 Optional, 

38 Sequence, 

39 Tuple, 

40 Type, 

41 Union, 

42) 

43 

44import lsst.utils 

45import lsst.utils.logging 

46from lsst.pex.config import ConfigurableField 

47from lsst.utils.timer import logInfo 

48 

49if TYPE_CHECKING: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true

50 from lsst.pex.config import Config 

51 

52try: 

53 import lsstDebug # type: ignore 

54except ImportError: 

55 lsstDebug = None 

56 

57from ._task_metadata import TaskMetadata 

58 

59# This defines the Python type to use for task metadata. It is a private 

60# class variable that can be accessed by other closely-related middleware 

61# code and test code. 

62_TASK_METADATA_TYPE = TaskMetadata 

63_TASK_FULL_METADATA_TYPE = TaskMetadata 

64 

65 

66class TaskError(Exception): 

67 """Use to report errors for which a traceback is not useful. 

68 

69 Notes 

70 ----- 

71 Examples of such errors: 

72 

73 - processCcd is asked to run detection, but not calibration, and no calexp 

74 is found. 

75 - coadd finds no valid images in the specified patch. 

76 """ 

77 

78 pass 

79 

80 

81class Task: 

82 r"""Base class for data processing tasks. 

83 

84 See :ref:`task-framework-overview` to learn what tasks are, and 

85 :ref:`creating-a-task` for more information about writing tasks. 

86 

87 Parameters 

88 ---------- 

89 config : `Task.ConfigClass` instance, optional 

90 Configuration for this task (an instance of Task.ConfigClass, which 

91 is a task-specific subclass of `lsst.pex.config.Config`, or `None`. 

92 If `None`: 

93 

94 - If parentTask specified then defaults to parentTask.config.\<name> 

95 - If parentTask is None then defaults to self.ConfigClass() 

96 

97 name : `str`, optional 

98 Brief name of task, or `None`; if `None` then defaults to 

99 `Task._DefaultName` 

100 parentTask : `Task`-type, optional 

101 The parent task of this subtask, if any. 

102 

103 - If `None` (a top-level task) then you must specify config and name 

104 is ignored. 

105 - If not `None` (a subtask) then you must specify name. 

106 log : `logging.Logger` or subclass, optional 

107 Log whose name is used as a log name prefix, or `None` for no prefix. 

108 Ignored if is parentTask specified, in which case 

109 ``parentTask.log``\ 's name is used as a prefix. The task's log name is 

110 ``prefix + "." + name`` if a prefix exists, else ``name``. The task's 

111 log is then a child logger of ``parentTask.log`` (if ``parentTask`` 

112 specified), or a child logger of the log from the argument 

113 (if ``log`` is not `None`). 

114 

115 Raises 

116 ------ 

117 RuntimeError 

118 Raised under these circumstances: 

119 

120 - If ``parentTask`` is `None` and ``config`` is `None`. 

121 - If ``parentTask`` is not `None` and ``name`` is `None`. 

122 - If ``name`` is `None` and ``_DefaultName`` does not exist. 

123 

124 Notes 

125 ----- 

126 The constructor must use keyword parameters for everything other than 

127 the ``config`` parameter which can be positional or use keyword form. 

128 

129 Useful attributes include: 

130 

131 - ``log``: an `logging.Logger` or subclass. 

132 - ``config``: task-specific configuration; an instance of ``ConfigClass`` 

133 (see below). 

134 - ``metadata``: a `TaskMetadata` for 

135 collecting task-specific metadata, e.g. data quality and performance 

136 metrics. This is data that is only meant to be persisted, never to be 

137 used by the task. 

138 

139 Use a `lsst.pipe.base.PipelineTask` subclass to perform I/O with a 

140 Butler. 

141 

142 Subclasses must also have an attribute ``ConfigClass`` that is a subclass 

143 of `lsst.pex.config.Config` which configures the task. Subclasses should 

144 also have an attribute ``_DefaultName``: the default name if there is no 

145 parent task. ``_DefaultName`` is required for subclasses of 

146 `~lsst.pipe.base.PipeLineTask` and recommended for subclasses of Task 

147 because it simplifies construction (e.g. for unit tests). 

148 """ 

149 

150 ConfigClass: ClassVar[Type[Config]] 

151 _DefaultName: ClassVar[str] 

152 

153 _add_module_logger_prefix: bool = True 

154 """Control whether the module prefix should be prepended to default 

155 logger names.""" 

156 

157 def __init__( 

158 self, 

159 config: Optional[Config] = None, 

160 *, 

161 name: Optional[str] = None, 

162 parentTask: Optional[Task] = None, 

163 log: Optional[Union[logging.Logger, lsst.utils.logging.LsstLogAdapter]] = None, 

164 ): 

165 self.metadata = _TASK_METADATA_TYPE() 

166 self.__parentTask: Optional[weakref.ReferenceType] 

167 self.__parentTask = parentTask if parentTask is None else weakref.ref(parentTask) 

168 

169 if parentTask is not None: 

170 if name is None: 

171 raise RuntimeError("name is required for a subtask") 

172 self._name = name 

173 self._fullName = parentTask._computeFullName(name) 

174 if config is None: 

175 config = getattr(parentTask.config, name) 

176 self._taskDict: Dict[str, weakref.ReferenceType[Task]] = parentTask._taskDict 

177 loggerName = parentTask.log.getChild(name).name 

178 else: 

179 if name is None: 

180 name = getattr(self, "_DefaultName", None) 

181 if name is None: 

182 raise RuntimeError("name is required for a task unless it has attribute _DefaultName") 

183 name = self._DefaultName 

184 self._name = name 

185 self._fullName = self._name 

186 if config is None: 

187 config = self.ConfigClass() 

188 self._taskDict = dict() 

189 loggerName = self._fullName 

190 if log is not None and log.name: 

191 loggerName = log.getChild(loggerName).name 

192 elif self._add_module_logger_prefix: 

193 # Prefix the logger name with the root module name. 

194 # We want all Task loggers to have this prefix to make 

195 # it easier to control them. This can be disabled by 

196 # a Task setting the class property _add_module_logger_prefix 

197 # to False -- in which case the logger name will not be 

198 # modified. 

199 module_name = self.__module__ 

200 module_root = module_name.split(".")[0] + "." 

201 if not loggerName.startswith(module_root): 

202 loggerName = module_root + loggerName 

203 

204 # Get a logger (that might be a subclass of logging.Logger). 

205 self.log: lsst.utils.logging.LsstLogAdapter = lsst.utils.logging.getLogger(loggerName) 

206 self.config: Config = config 

207 self.config.validate() 

208 if lsstDebug: 

209 self._display = lsstDebug.Info(self.__module__).display 

210 else: 

211 self._display = None 

212 self._taskDict[self._fullName] = weakref.ref(self) 

213 

214 @property 

215 def _parentTask(self) -> Optional[Task]: 

216 return self.__parentTask if self.__parentTask is None else self.__parentTask() 

217 

218 def emptyMetadata(self) -> None: 

219 """Empty (clear) the metadata for this Task and all sub-Tasks.""" 

220 for wref in self._taskDict.values(): 

221 subtask = wref() 

222 assert subtask is not None, "Unexpected garbage collection of subtask." 

223 subtask.metadata = _TASK_METADATA_TYPE() 

224 

225 def getFullMetadata(self) -> TaskMetadata: 

226 """Get metadata for all tasks. 

227 

228 Returns 

229 ------- 

230 metadata : `TaskMetadata` 

231 The keys are the full task name. 

232 Values are metadata for the top-level task and all subtasks, 

233 sub-subtasks, etc. 

234 

235 Notes 

236 ----- 

237 The returned metadata includes timing information (if 

238 ``@timer.timeMethod`` is used) and any metadata set by the task. The 

239 name of each item consists of the full task name with ``.`` replaced 

240 by ``:``, followed by ``.`` and the name of the item, e.g.:: 

241 

242 topLevelTaskName:subtaskName:subsubtaskName.itemName 

243 

244 using ``:`` in the full task name disambiguates the rare situation 

245 that a task has a subtask and a metadata item with the same name. 

246 """ 

247 fullMetadata = _TASK_FULL_METADATA_TYPE() 

248 for fullName, wref in self.getTaskDict().items(): 

249 subtask = wref() 

250 assert subtask is not None, "Unexpected garbage collection of subtask." 

251 fullMetadata[fullName.replace(".", ":")] = subtask.metadata 

252 return fullMetadata 

253 

254 def getFullName(self) -> str: 

255 """Get the task name as a hierarchical name including parent task 

256 names. 

257 

258 Returns 

259 ------- 

260 fullName : `str` 

261 The full name consists of the name of the parent task and each 

262 subtask separated by periods. For example: 

263 

264 - The full name of top-level task "top" is simply "top". 

265 - The full name of subtask "sub" of top-level task "top" is 

266 "top.sub". 

267 - The full name of subtask "sub2" of subtask "sub" of top-level 

268 task "top" is "top.sub.sub2". 

269 """ 

270 return self._fullName 

271 

272 def getName(self) -> str: 

273 """Get the name of the task. 

274 

275 Returns 

276 ------- 

277 taskName : `str` 

278 Name of the task. 

279 

280 See also 

281 -------- 

282 getFullName 

283 """ 

284 return self._name 

285 

286 def getTaskDict(self) -> Dict[str, weakref.ReferenceType[Task]]: 

287 """Get a dictionary of all tasks as a shallow copy. 

288 

289 Returns 

290 ------- 

291 taskDict : `dict` 

292 Dictionary containing full task name: task object for the top-level 

293 task and all subtasks, sub-subtasks, etc. 

294 """ 

295 return self._taskDict.copy() 

296 

297 def makeSubtask(self, name: str, **keyArgs: Any) -> None: 

298 """Create a subtask as a new instance as the ``name`` attribute of this 

299 task. 

300 

301 Parameters 

302 ---------- 

303 name : `str` 

304 Brief name of the subtask. 

305 keyArgs 

306 Extra keyword arguments used to construct the task. The following 

307 arguments are automatically provided and cannot be overridden: 

308 

309 - "config". 

310 - "parentTask". 

311 

312 Notes 

313 ----- 

314 The subtask must be defined by ``Task.config.name``, an instance of 

315 `~lsst.pex.config.ConfigurableField` or 

316 `~lsst.pex.config.RegistryField`. 

317 """ 

318 taskField = getattr(self.config, name, None) 

319 if taskField is None: 

320 raise KeyError(f"{self.getFullName()}'s config does not have field {name!r}") 

321 subtask = taskField.apply(name=name, parentTask=self, **keyArgs) 

322 setattr(self, name, subtask) 

323 

324 @contextlib.contextmanager 

325 def timer(self, name: str, logLevel: int = logging.DEBUG) -> Iterator[None]: 

326 """Context manager to log performance data for an arbitrary block of 

327 code. 

328 

329 Parameters 

330 ---------- 

331 name : `str` 

332 Name of code being timed; data will be logged using item name: 

333 ``Start`` and ``End``. 

334 logLevel 

335 A `logging` level constant. 

336 

337 Examples 

338 -------- 

339 Creating a timer context: 

340 

341 .. code-block:: python 

342 

343 with self.timer("someCodeToTime"): 

344 pass # code to time 

345 

346 See also 

347 -------- 

348 timer.logInfo 

349 """ 

350 logInfo(obj=self, prefix=name + "Start", logLevel=logLevel) 

351 try: 

352 yield 

353 finally: 

354 logInfo(obj=self, prefix=name + "End", logLevel=logLevel) 

355 

356 @classmethod 

357 def makeField(cls, doc: str) -> ConfigurableField: 

358 """Make a `lsst.pex.config.ConfigurableField` for this task. 

359 

360 Parameters 

361 ---------- 

362 doc : `str` 

363 Help text for the field. 

364 

365 Returns 

366 ------- 

367 configurableField : `lsst.pex.config.ConfigurableField` 

368 A `~ConfigurableField` for this task. 

369 

370 Examples 

371 -------- 

372 Provides a convenient way to specify this task is a subtask of another 

373 task. 

374 

375 Here is an example of use: 

376 

377 .. code-block:: python 

378 

379 class OtherTaskConfig(lsst.pex.config.Config): 

380 aSubtask = ATaskClass.makeField("brief description of task") 

381 """ 

382 return ConfigurableField(doc=doc, target=cls) 

383 

384 def _computeFullName(self, name: str) -> str: 

385 """Compute the full name of a subtask or metadata item, given its brief 

386 name. 

387 

388 Parameters 

389 ---------- 

390 name : `str` 

391 Brief name of subtask or metadata item. 

392 

393 Returns 

394 ------- 

395 fullName : `str` 

396 The full name: the ``name`` argument prefixed by the full task name 

397 and a period. 

398 

399 Notes 

400 ----- 

401 For example: if the full name of this task is "top.sub.sub2" 

402 then ``_computeFullName("subname")`` returns 

403 ``"top.sub.sub2.subname"``. 

404 """ 

405 return f"{self._fullName}.{name}" 

406 

407 @staticmethod 

408 def _unpickle_via_factory( 

409 factory: Callable[..., Task], args: Sequence[Any], kwargs: Dict[str, Any] 

410 ) -> Task: 

411 """Unpickle something by calling a factory 

412 

413 Allows subclasses to unpickle using `__reduce__` with keyword 

414 arguments as well as positional arguments. 

415 """ 

416 return factory(*args, **kwargs) 

417 

418 def _reduce_kwargs(self) -> Dict[str, Any]: 

419 """Returns a dict of the keyword arguments that should be used 

420 by `__reduce__`. 

421 

422 Subclasses with additional arguments should always call the parent 

423 class method to ensure that the standard parameters are included. 

424 

425 Returns 

426 ------- 

427 kwargs : `dict` 

428 Keyword arguments to be used when pickling. 

429 """ 

430 return dict( 

431 config=self.config, 

432 name=self._name, 

433 parentTask=self._parentTask, 

434 ) 

435 

436 def __reduce__( 

437 self, 

438 ) -> Tuple[ 

439 Callable[[Callable[..., Task], Sequence[Any], Dict[str, Any]], Task], 

440 Tuple[Type[Task], Sequence[Any], Dict[str, Any]], 

441 ]: 

442 """Pickler.""" 

443 return self._unpickle_via_factory, (self.__class__, [], self._reduce_kwargs())