Coverage for python/lsst/pipe/base/task.py: 31%
109 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-15 02:04 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-15 02:04 -0800
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23from __future__ import annotations
25__all__ = ["Task", "TaskError"]
27import contextlib
28import logging
29import weakref
30from typing import (
31 TYPE_CHECKING,
32 Any,
33 Callable,
34 ClassVar,
35 Dict,
36 Iterator,
37 Optional,
38 Sequence,
39 Tuple,
40 Type,
41 Union,
42)
44import lsst.utils
45import lsst.utils.logging
46from lsst.pex.config import ConfigurableField
47from lsst.utils.timer import logInfo
49if TYPE_CHECKING: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true
50 from lsst.pex.config import Config
52try:
53 import lsstDebug # type: ignore
54except ImportError:
55 lsstDebug = None
57from ._task_metadata import TaskMetadata
59# This defines the Python type to use for task metadata. It is a private
60# class variable that can be accessed by other closely-related middleware
61# code and test code.
62_TASK_METADATA_TYPE = TaskMetadata
63_TASK_FULL_METADATA_TYPE = TaskMetadata
66class TaskError(Exception):
67 """Use to report errors for which a traceback is not useful.
69 Notes
70 -----
71 Examples of such errors:
73 - processCcd is asked to run detection, but not calibration, and no calexp
74 is found.
75 - coadd finds no valid images in the specified patch.
76 """
78 pass
81class Task:
82 r"""Base class for data processing tasks.
84 See :ref:`task-framework-overview` to learn what tasks are, and
85 :ref:`creating-a-task` for more information about writing tasks.
87 Parameters
88 ----------
89 config : `Task.ConfigClass` instance, optional
90 Configuration for this task (an instance of Task.ConfigClass, which
91 is a task-specific subclass of `lsst.pex.config.Config`, or `None`.
92 If `None`:
94 - If parentTask specified then defaults to parentTask.config.\<name>
95 - If parentTask is None then defaults to self.ConfigClass()
97 name : `str`, optional
98 Brief name of task, or `None`; if `None` then defaults to
99 `Task._DefaultName`
100 parentTask : `Task`-type, optional
101 The parent task of this subtask, if any.
103 - If `None` (a top-level task) then you must specify config and name
104 is ignored.
105 - If not `None` (a subtask) then you must specify name.
106 log : `logging.Logger` or subclass, optional
107 Log whose name is used as a log name prefix, or `None` for no prefix.
108 Ignored if is parentTask specified, in which case
109 ``parentTask.log``\ 's name is used as a prefix. The task's log name is
110 ``prefix + "." + name`` if a prefix exists, else ``name``. The task's
111 log is then a child logger of ``parentTask.log`` (if ``parentTask``
112 specified), or a child logger of the log from the argument
113 (if ``log`` is not `None`).
115 Raises
116 ------
117 RuntimeError
118 Raised under these circumstances:
120 - If ``parentTask`` is `None` and ``config`` is `None`.
121 - If ``parentTask`` is not `None` and ``name`` is `None`.
122 - If ``name`` is `None` and ``_DefaultName`` does not exist.
124 Notes
125 -----
126 Useful attributes include:
128 - ``log``: an `logging.Logger` or subclass.
129 - ``config``: task-specific configuration; an instance of ``ConfigClass``
130 (see below).
131 - ``metadata``: a `TaskMetadata` for
132 collecting task-specific metadata, e.g. data quality and performance
133 metrics. This is data that is only meant to be persisted, never to be
134 used by the task.
136 Use a `lsst.pipe.base.PipelineTask` subclass to perform I/O with a
137 Butler.
139 Subclasses must also have an attribute ``ConfigClass`` that is a subclass
140 of `lsst.pex.config.Config` which configures the task. Subclasses should
141 also have an attribute ``_DefaultName``: the default name if there is no
142 parent task. ``_DefaultName`` is required for subclasses of
143 `~lsst.pipe.base.PipeLineTask` and recommended for subclasses of Task
144 because it simplifies construction (e.g. for unit tests).
145 """
147 ConfigClass: ClassVar[Type[Config]]
148 _DefaultName: ClassVar[str]
150 _add_module_logger_prefix: bool = True
151 """Control whether the module prefix should be prepended to default
152 logger names."""
154 def __init__(
155 self,
156 config: Optional[Config] = None,
157 name: Optional[str] = None,
158 parentTask: Optional[Task] = None,
159 log: Optional[Union[logging.Logger, lsst.utils.logging.LsstLogAdapter]] = None,
160 ):
161 self.metadata = _TASK_METADATA_TYPE()
162 self.__parentTask: Optional[weakref.ReferenceType]
163 self.__parentTask = parentTask if parentTask is None else weakref.ref(parentTask)
165 if parentTask is not None:
166 if name is None:
167 raise RuntimeError("name is required for a subtask")
168 self._name = name
169 self._fullName = parentTask._computeFullName(name)
170 if config is None:
171 config = getattr(parentTask.config, name)
172 self._taskDict: Dict[str, weakref.ReferenceType[Task]] = parentTask._taskDict
173 loggerName = parentTask.log.getChild(name).name
174 else:
175 if name is None:
176 name = getattr(self, "_DefaultName", None)
177 if name is None:
178 raise RuntimeError("name is required for a task unless it has attribute _DefaultName")
179 name = self._DefaultName
180 self._name = name
181 self._fullName = self._name
182 if config is None:
183 config = self.ConfigClass()
184 self._taskDict = dict()
185 loggerName = self._fullName
186 if log is not None and log.name:
187 loggerName = log.getChild(loggerName).name
188 elif self._add_module_logger_prefix:
189 # Prefix the logger name with the root module name.
190 # We want all Task loggers to have this prefix to make
191 # it easier to control them. This can be disabled by
192 # a Task setting the class property _add_module_logger_prefix
193 # to False -- in which case the logger name will not be
194 # modified.
195 module_name = self.__module__
196 module_root = module_name.split(".")[0] + "."
197 if not loggerName.startswith(module_root):
198 loggerName = module_root + loggerName
200 # Get a logger (that might be a subclass of logging.Logger).
201 self.log: lsst.utils.logging.LsstLogAdapter = lsst.utils.logging.getLogger(loggerName)
202 self.config: Config = config
203 self.config.validate()
204 if lsstDebug:
205 self._display = lsstDebug.Info(self.__module__).display
206 else:
207 self._display = None
208 self._taskDict[self._fullName] = weakref.ref(self)
210 @property
211 def _parentTask(self) -> Optional[Task]:
212 return self.__parentTask if self.__parentTask is None else self.__parentTask()
214 def emptyMetadata(self) -> None:
215 """Empty (clear) the metadata for this Task and all sub-Tasks."""
216 for wref in self._taskDict.values():
217 subtask = wref()
218 assert subtask is not None, "Unexpected garbage collection of subtask."
219 subtask.metadata = _TASK_METADATA_TYPE()
221 def getFullMetadata(self) -> TaskMetadata:
222 """Get metadata for all tasks.
224 Returns
225 -------
226 metadata : `TaskMetadata`
227 The keys are the full task name.
228 Values are metadata for the top-level task and all subtasks,
229 sub-subtasks, etc.
231 Notes
232 -----
233 The returned metadata includes timing information (if
234 ``@timer.timeMethod`` is used) and any metadata set by the task. The
235 name of each item consists of the full task name with ``.`` replaced
236 by ``:``, followed by ``.`` and the name of the item, e.g.::
238 topLevelTaskName:subtaskName:subsubtaskName.itemName
240 using ``:`` in the full task name disambiguates the rare situation
241 that a task has a subtask and a metadata item with the same name.
242 """
243 fullMetadata = _TASK_FULL_METADATA_TYPE()
244 for fullName, wref in self.getTaskDict().items():
245 subtask = wref()
246 assert subtask is not None, "Unexpected garbage collection of subtask."
247 fullMetadata[fullName.replace(".", ":")] = subtask.metadata
248 return fullMetadata
250 def getFullName(self) -> str:
251 """Get the task name as a hierarchical name including parent task
252 names.
254 Returns
255 -------
256 fullName : `str`
257 The full name consists of the name of the parent task and each
258 subtask separated by periods. For example:
260 - The full name of top-level task "top" is simply "top".
261 - The full name of subtask "sub" of top-level task "top" is
262 "top.sub".
263 - The full name of subtask "sub2" of subtask "sub" of top-level
264 task "top" is "top.sub.sub2".
265 """
266 return self._fullName
268 def getName(self) -> str:
269 """Get the name of the task.
271 Returns
272 -------
273 taskName : `str`
274 Name of the task.
276 See also
277 --------
278 getFullName
279 """
280 return self._name
282 def getTaskDict(self) -> Dict[str, weakref.ReferenceType[Task]]:
283 """Get a dictionary of all tasks as a shallow copy.
285 Returns
286 -------
287 taskDict : `dict`
288 Dictionary containing full task name: task object for the top-level
289 task and all subtasks, sub-subtasks, etc.
290 """
291 return self._taskDict.copy()
293 def makeSubtask(self, name: str, **keyArgs: Any) -> None:
294 """Create a subtask as a new instance as the ``name`` attribute of this
295 task.
297 Parameters
298 ----------
299 name : `str`
300 Brief name of the subtask.
301 keyArgs
302 Extra keyword arguments used to construct the task. The following
303 arguments are automatically provided and cannot be overridden:
305 - "config".
306 - "parentTask".
308 Notes
309 -----
310 The subtask must be defined by ``Task.config.name``, an instance of
311 `~lsst.pex.config.ConfigurableField` or
312 `~lsst.pex.config.RegistryField`.
313 """
314 taskField = getattr(self.config, name, None)
315 if taskField is None:
316 raise KeyError(f"{self.getFullName()}'s config does not have field {name!r}")
317 subtask = taskField.apply(name=name, parentTask=self, **keyArgs)
318 setattr(self, name, subtask)
320 @contextlib.contextmanager
321 def timer(self, name: str, logLevel: int = logging.DEBUG) -> Iterator[None]:
322 """Context manager to log performance data for an arbitrary block of
323 code.
325 Parameters
326 ----------
327 name : `str`
328 Name of code being timed; data will be logged using item name:
329 ``Start`` and ``End``.
330 logLevel
331 A `logging` level constant.
333 Examples
334 --------
335 Creating a timer context:
337 .. code-block:: python
339 with self.timer("someCodeToTime"):
340 pass # code to time
342 See also
343 --------
344 timer.logInfo
345 """
346 logInfo(obj=self, prefix=name + "Start", logLevel=logLevel)
347 try:
348 yield
349 finally:
350 logInfo(obj=self, prefix=name + "End", logLevel=logLevel)
352 @classmethod
353 def makeField(cls, doc: str) -> ConfigurableField:
354 """Make a `lsst.pex.config.ConfigurableField` for this task.
356 Parameters
357 ----------
358 doc : `str`
359 Help text for the field.
361 Returns
362 -------
363 configurableField : `lsst.pex.config.ConfigurableField`
364 A `~ConfigurableField` for this task.
366 Examples
367 --------
368 Provides a convenient way to specify this task is a subtask of another
369 task.
371 Here is an example of use:
373 .. code-block:: python
375 class OtherTaskConfig(lsst.pex.config.Config):
376 aSubtask = ATaskClass.makeField("brief description of task")
377 """
378 return ConfigurableField(doc=doc, target=cls)
380 def _computeFullName(self, name: str) -> str:
381 """Compute the full name of a subtask or metadata item, given its brief
382 name.
384 Parameters
385 ----------
386 name : `str`
387 Brief name of subtask or metadata item.
389 Returns
390 -------
391 fullName : `str`
392 The full name: the ``name`` argument prefixed by the full task name
393 and a period.
395 Notes
396 -----
397 For example: if the full name of this task is "top.sub.sub2"
398 then ``_computeFullName("subname")`` returns
399 ``"top.sub.sub2.subname"``.
400 """
401 return f"{self._fullName}.{name}"
403 @staticmethod
404 def _unpickle_via_factory(
405 factory: Callable[..., Task], args: Sequence[Any], kwargs: Dict[str, Any]
406 ) -> Task:
407 """Unpickle something by calling a factory
409 Allows subclasses to unpickle using `__reduce__` with keyword
410 arguments as well as positional arguments.
411 """
412 return factory(*args, **kwargs)
414 def _reduce_kwargs(self) -> Dict[str, Any]:
415 """Returns a dict of the keyword arguments that should be used
416 by `__reduce__`.
418 Subclasses with additional arguments should always call the parent
419 class method to ensure that the standard parameters are included.
421 Returns
422 -------
423 kwargs : `dict`
424 Keyword arguments to be used when pickling.
425 """
426 return dict(
427 config=self.config,
428 name=self._name,
429 parentTask=self._parentTask,
430 )
432 def __reduce__(
433 self,
434 ) -> Tuple[
435 Callable[[Callable[..., Task], Sequence[Any], Dict[str, Any]], Task],
436 Tuple[Type[Task], Sequence[Any], Dict[str, Any]],
437 ]:
438 """Pickler."""
439 return self._unpickle_via_factory, (self.__class__, [], self._reduce_kwargs())