Coverage for python/lsst/pipe/base/task.py: 31%

109 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-09 11:18 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23from __future__ import annotations 

24 

25__all__ = ["Task", "TaskError"] 

26 

27import contextlib 

28import logging 

29import weakref 

30from typing import ( 

31 TYPE_CHECKING, 

32 Any, 

33 Callable, 

34 ClassVar, 

35 Dict, 

36 Iterator, 

37 Optional, 

38 Sequence, 

39 Tuple, 

40 Type, 

41 Union, 

42) 

43 

44import lsst.utils 

45import lsst.utils.logging 

46from lsst.pex.config import ConfigurableField 

47from lsst.utils.timer import logInfo 

48 

49if TYPE_CHECKING: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true

50 from lsst.pex.config import Config 

51 

52try: 

53 import lsstDebug # type: ignore 

54except ImportError: 

55 lsstDebug = None 

56 

57from ._task_metadata import TaskMetadata 

58 

59# This defines the Python type to use for task metadata. It is a private 

60# class variable that can be accessed by other closely-related middleware 

61# code and test code. 

62_TASK_METADATA_TYPE = TaskMetadata 

63_TASK_FULL_METADATA_TYPE = TaskMetadata 

64 

65 

66class TaskError(Exception): 

67 """Use to report errors for which a traceback is not useful. 

68 

69 Notes 

70 ----- 

71 Examples of such errors: 

72 

73 - processCcd is asked to run detection, but not calibration, and no calexp 

74 is found. 

75 - coadd finds no valid images in the specified patch. 

76 """ 

77 

78 pass 

79 

80 

81class Task: 

82 r"""Base class for data processing tasks. 

83 

84 See :ref:`task-framework-overview` to learn what tasks are, and 

85 :ref:`creating-a-task` for more information about writing tasks. 

86 

87 Parameters 

88 ---------- 

89 config : `Task.ConfigClass` instance, optional 

90 Configuration for this task (an instance of Task.ConfigClass, which 

91 is a task-specific subclass of `lsst.pex.config.Config`, or `None`. 

92 If `None`: 

93 

94 - If parentTask specified then defaults to parentTask.config.\<name> 

95 - If parentTask is None then defaults to self.ConfigClass() 

96 

97 name : `str`, optional 

98 Brief name of task, or `None`; if `None` then defaults to 

99 `Task._DefaultName` 

100 parentTask : `Task`-type, optional 

101 The parent task of this subtask, if any. 

102 

103 - If `None` (a top-level task) then you must specify config and name 

104 is ignored. 

105 - If not `None` (a subtask) then you must specify name. 

106 log : `logging.Logger` or subclass, optional 

107 Log whose name is used as a log name prefix, or `None` for no prefix. 

108 Ignored if is parentTask specified, in which case 

109 ``parentTask.log``\ 's name is used as a prefix. The task's log name is 

110 ``prefix + "." + name`` if a prefix exists, else ``name``. The task's 

111 log is then a child logger of ``parentTask.log`` (if ``parentTask`` 

112 specified), or a child logger of the log from the argument 

113 (if ``log`` is not `None`). 

114 

115 Raises 

116 ------ 

117 RuntimeError 

118 Raised under these circumstances: 

119 

120 - If ``parentTask`` is `None` and ``config`` is `None`. 

121 - If ``parentTask`` is not `None` and ``name`` is `None`. 

122 - If ``name`` is `None` and ``_DefaultName`` does not exist. 

123 

124 Notes 

125 ----- 

126 Useful attributes include: 

127 

128 - ``log``: an `logging.Logger` or subclass. 

129 - ``config``: task-specific configuration; an instance of ``ConfigClass`` 

130 (see below). 

131 - ``metadata``: a `TaskMetadata` for 

132 collecting task-specific metadata, e.g. data quality and performance 

133 metrics. This is data that is only meant to be persisted, never to be 

134 used by the task. 

135 

136 Use a `lsst.pipe.base.PipelineTask` subclass to perform I/O with a 

137 Butler. 

138 

139 Subclasses must also have an attribute ``ConfigClass`` that is a subclass 

140 of `lsst.pex.config.Config` which configures the task. Subclasses should 

141 also have an attribute ``_DefaultName``: the default name if there is no 

142 parent task. ``_DefaultName`` is required for subclasses of 

143 `~lsst.pipe.base.PipeLineTask` and recommended for subclasses of Task 

144 because it simplifies construction (e.g. for unit tests). 

145 """ 

146 

147 ConfigClass: ClassVar[Type[Config]] 

148 _DefaultName: ClassVar[str] 

149 

150 _add_module_logger_prefix: bool = True 

151 """Control whether the module prefix should be prepended to default 

152 logger names.""" 

153 

154 def __init__( 

155 self, 

156 config: Optional[Config] = None, 

157 name: Optional[str] = None, 

158 parentTask: Optional[Task] = None, 

159 log: Optional[Union[logging.Logger, lsst.utils.logging.LsstLogAdapter]] = None, 

160 ): 

161 self.metadata = _TASK_METADATA_TYPE() 

162 self.__parentTask: Optional[weakref.ReferenceType] 

163 self.__parentTask = parentTask if parentTask is None else weakref.ref(parentTask) 

164 

165 if parentTask is not None: 

166 if name is None: 

167 raise RuntimeError("name is required for a subtask") 

168 self._name = name 

169 self._fullName = parentTask._computeFullName(name) 

170 if config is None: 

171 config = getattr(parentTask.config, name) 

172 self._taskDict: Dict[str, weakref.ReferenceType[Task]] = parentTask._taskDict 

173 loggerName = parentTask.log.getChild(name).name 

174 else: 

175 if name is None: 

176 name = getattr(self, "_DefaultName", None) 

177 if name is None: 

178 raise RuntimeError("name is required for a task unless it has attribute _DefaultName") 

179 name = self._DefaultName 

180 self._name = name 

181 self._fullName = self._name 

182 if config is None: 

183 config = self.ConfigClass() 

184 self._taskDict = dict() 

185 loggerName = self._fullName 

186 if log is not None and log.name: 

187 loggerName = log.getChild(loggerName).name 

188 elif self._add_module_logger_prefix: 

189 # Prefix the logger name with the root module name. 

190 # We want all Task loggers to have this prefix to make 

191 # it easier to control them. This can be disabled by 

192 # a Task setting the class property _add_module_logger_prefix 

193 # to False -- in which case the logger name will not be 

194 # modified. 

195 module_name = self.__module__ 

196 module_root = module_name.split(".")[0] + "." 

197 if not loggerName.startswith(module_root): 

198 loggerName = module_root + loggerName 

199 

200 # Get a logger (that might be a subclass of logging.Logger). 

201 self.log: lsst.utils.logging.LsstLogAdapter = lsst.utils.logging.getLogger(loggerName) 

202 self.config: Config = config 

203 self.config.validate() 

204 if lsstDebug: 

205 self._display = lsstDebug.Info(self.__module__).display 

206 else: 

207 self._display = None 

208 self._taskDict[self._fullName] = weakref.ref(self) 

209 

210 @property 

211 def _parentTask(self) -> Optional[Task]: 

212 return self.__parentTask if self.__parentTask is None else self.__parentTask() 

213 

214 def emptyMetadata(self) -> None: 

215 """Empty (clear) the metadata for this Task and all sub-Tasks.""" 

216 for wref in self._taskDict.values(): 

217 subtask = wref() 

218 assert subtask is not None, "Unexpected garbage collection of subtask." 

219 subtask.metadata = _TASK_METADATA_TYPE() 

220 

221 def getFullMetadata(self) -> TaskMetadata: 

222 """Get metadata for all tasks. 

223 

224 Returns 

225 ------- 

226 metadata : `TaskMetadata` 

227 The keys are the full task name. 

228 Values are metadata for the top-level task and all subtasks, 

229 sub-subtasks, etc. 

230 

231 Notes 

232 ----- 

233 The returned metadata includes timing information (if 

234 ``@timer.timeMethod`` is used) and any metadata set by the task. The 

235 name of each item consists of the full task name with ``.`` replaced 

236 by ``:``, followed by ``.`` and the name of the item, e.g.:: 

237 

238 topLevelTaskName:subtaskName:subsubtaskName.itemName 

239 

240 using ``:`` in the full task name disambiguates the rare situation 

241 that a task has a subtask and a metadata item with the same name. 

242 """ 

243 fullMetadata = _TASK_FULL_METADATA_TYPE() 

244 for fullName, wref in self.getTaskDict().items(): 

245 subtask = wref() 

246 assert subtask is not None, "Unexpected garbage collection of subtask." 

247 fullMetadata[fullName.replace(".", ":")] = subtask.metadata 

248 return fullMetadata 

249 

250 def getFullName(self) -> str: 

251 """Get the task name as a hierarchical name including parent task 

252 names. 

253 

254 Returns 

255 ------- 

256 fullName : `str` 

257 The full name consists of the name of the parent task and each 

258 subtask separated by periods. For example: 

259 

260 - The full name of top-level task "top" is simply "top". 

261 - The full name of subtask "sub" of top-level task "top" is 

262 "top.sub". 

263 - The full name of subtask "sub2" of subtask "sub" of top-level 

264 task "top" is "top.sub.sub2". 

265 """ 

266 return self._fullName 

267 

268 def getName(self) -> str: 

269 """Get the name of the task. 

270 

271 Returns 

272 ------- 

273 taskName : `str` 

274 Name of the task. 

275 

276 See also 

277 -------- 

278 getFullName 

279 """ 

280 return self._name 

281 

282 def getTaskDict(self) -> Dict[str, weakref.ReferenceType[Task]]: 

283 """Get a dictionary of all tasks as a shallow copy. 

284 

285 Returns 

286 ------- 

287 taskDict : `dict` 

288 Dictionary containing full task name: task object for the top-level 

289 task and all subtasks, sub-subtasks, etc. 

290 """ 

291 return self._taskDict.copy() 

292 

293 def makeSubtask(self, name: str, **keyArgs: Any) -> None: 

294 """Create a subtask as a new instance as the ``name`` attribute of this 

295 task. 

296 

297 Parameters 

298 ---------- 

299 name : `str` 

300 Brief name of the subtask. 

301 keyArgs 

302 Extra keyword arguments used to construct the task. The following 

303 arguments are automatically provided and cannot be overridden: 

304 

305 - "config". 

306 - "parentTask". 

307 

308 Notes 

309 ----- 

310 The subtask must be defined by ``Task.config.name``, an instance of 

311 `~lsst.pex.config.ConfigurableField` or 

312 `~lsst.pex.config.RegistryField`. 

313 """ 

314 taskField = getattr(self.config, name, None) 

315 if taskField is None: 

316 raise KeyError(f"{self.getFullName()}'s config does not have field {name!r}") 

317 subtask = taskField.apply(name=name, parentTask=self, **keyArgs) 

318 setattr(self, name, subtask) 

319 

320 @contextlib.contextmanager 

321 def timer(self, name: str, logLevel: int = logging.DEBUG) -> Iterator[None]: 

322 """Context manager to log performance data for an arbitrary block of 

323 code. 

324 

325 Parameters 

326 ---------- 

327 name : `str` 

328 Name of code being timed; data will be logged using item name: 

329 ``Start`` and ``End``. 

330 logLevel 

331 A `logging` level constant. 

332 

333 Examples 

334 -------- 

335 Creating a timer context: 

336 

337 .. code-block:: python 

338 

339 with self.timer("someCodeToTime"): 

340 pass # code to time 

341 

342 See also 

343 -------- 

344 timer.logInfo 

345 """ 

346 logInfo(obj=self, prefix=name + "Start", logLevel=logLevel) 

347 try: 

348 yield 

349 finally: 

350 logInfo(obj=self, prefix=name + "End", logLevel=logLevel) 

351 

352 @classmethod 

353 def makeField(cls, doc: str) -> ConfigurableField: 

354 """Make a `lsst.pex.config.ConfigurableField` for this task. 

355 

356 Parameters 

357 ---------- 

358 doc : `str` 

359 Help text for the field. 

360 

361 Returns 

362 ------- 

363 configurableField : `lsst.pex.config.ConfigurableField` 

364 A `~ConfigurableField` for this task. 

365 

366 Examples 

367 -------- 

368 Provides a convenient way to specify this task is a subtask of another 

369 task. 

370 

371 Here is an example of use: 

372 

373 .. code-block:: python 

374 

375 class OtherTaskConfig(lsst.pex.config.Config): 

376 aSubtask = ATaskClass.makeField("brief description of task") 

377 """ 

378 return ConfigurableField(doc=doc, target=cls) 

379 

380 def _computeFullName(self, name: str) -> str: 

381 """Compute the full name of a subtask or metadata item, given its brief 

382 name. 

383 

384 Parameters 

385 ---------- 

386 name : `str` 

387 Brief name of subtask or metadata item. 

388 

389 Returns 

390 ------- 

391 fullName : `str` 

392 The full name: the ``name`` argument prefixed by the full task name 

393 and a period. 

394 

395 Notes 

396 ----- 

397 For example: if the full name of this task is "top.sub.sub2" 

398 then ``_computeFullName("subname")`` returns 

399 ``"top.sub.sub2.subname"``. 

400 """ 

401 return f"{self._fullName}.{name}" 

402 

403 @staticmethod 

404 def _unpickle_via_factory( 

405 factory: Callable[..., Task], args: Sequence[Any], kwargs: Dict[str, Any] 

406 ) -> Task: 

407 """Unpickle something by calling a factory 

408 

409 Allows subclasses to unpickle using `__reduce__` with keyword 

410 arguments as well as positional arguments. 

411 """ 

412 return factory(*args, **kwargs) 

413 

414 def _reduce_kwargs(self) -> Dict[str, Any]: 

415 """Returns a dict of the keyword arguments that should be used 

416 by `__reduce__`. 

417 

418 Subclasses with additional arguments should always call the parent 

419 class method to ensure that the standard parameters are included. 

420 

421 Returns 

422 ------- 

423 kwargs : `dict` 

424 Keyword arguments to be used when pickling. 

425 """ 

426 return dict( 

427 config=self.config, 

428 name=self._name, 

429 parentTask=self._parentTask, 

430 ) 

431 

432 def __reduce__( 

433 self, 

434 ) -> Tuple[ 

435 Callable[[Callable[..., Task], Sequence[Any], Dict[str, Any]], Task], 

436 Tuple[Type[Task], Sequence[Any], Dict[str, Any]], 

437 ]: 

438 """Pickler.""" 

439 return self._unpickle_via_factory, (self.__class__, [], self._reduce_kwargs())