Coverage for python/lsst/pipe/base/task.py: 31%
109 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-06 02:42 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-06 02:42 -0700
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23from __future__ import annotations
25__all__ = ["Task", "TaskError"]
27import contextlib
28import logging
29import weakref
30from typing import (
31 TYPE_CHECKING,
32 Any,
33 Callable,
34 ClassVar,
35 Dict,
36 Iterator,
37 Optional,
38 Sequence,
39 Tuple,
40 Type,
41 Union,
42)
44import lsst.utils
45import lsst.utils.logging
46from lsst.pex.config import ConfigurableField
47from lsst.utils.timer import logInfo
49if TYPE_CHECKING: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true
50 from lsst.pex.config import Config
52try:
53 import lsstDebug # type: ignore
54except ImportError:
55 lsstDebug = None
57from ._task_metadata import TaskMetadata
59# This defines the Python type to use for task metadata. It is a private
60# class variable that can be accessed by other closely-related middleware
61# code and test code.
62_TASK_METADATA_TYPE = TaskMetadata
63_TASK_FULL_METADATA_TYPE = TaskMetadata
66class TaskError(Exception):
67 """Use to report errors for which a traceback is not useful.
69 Notes
70 -----
71 Examples of such errors:
73 - processCcd is asked to run detection, but not calibration, and no calexp
74 is found.
75 - coadd finds no valid images in the specified patch.
76 """
78 pass
81class Task:
82 r"""Base class for data processing tasks.
84 See :ref:`task-framework-overview` to learn what tasks are, and
85 :ref:`creating-a-task` for more information about writing tasks.
87 Parameters
88 ----------
89 config : `Task.ConfigClass` instance, optional
90 Configuration for this task (an instance of Task.ConfigClass, which
91 is a task-specific subclass of `lsst.pex.config.Config`, or `None`.
92 If `None`:
94 - If parentTask specified then defaults to parentTask.config.\<name>
95 - If parentTask is None then defaults to self.ConfigClass()
97 name : `str`, optional
98 Brief name of task, or `None`; if `None` then defaults to
99 `Task._DefaultName`
100 parentTask : `Task`-type, optional
101 The parent task of this subtask, if any.
103 - If `None` (a top-level task) then you must specify config and name
104 is ignored.
105 - If not `None` (a subtask) then you must specify name.
106 log : `logging.Logger` or subclass, optional
107 Log whose name is used as a log name prefix, or `None` for no prefix.
108 Ignored if is parentTask specified, in which case
109 ``parentTask.log``\ 's name is used as a prefix. The task's log name is
110 ``prefix + "." + name`` if a prefix exists, else ``name``. The task's
111 log is then a child logger of ``parentTask.log`` (if ``parentTask``
112 specified), or a child logger of the log from the argument
113 (if ``log`` is not `None`).
115 Raises
116 ------
117 RuntimeError
118 Raised under these circumstances:
120 - If ``parentTask`` is `None` and ``config`` is `None`.
121 - If ``parentTask`` is not `None` and ``name`` is `None`.
122 - If ``name`` is `None` and ``_DefaultName`` does not exist.
124 Notes
125 -----
126 The constructor must use keyword parameters for everything other than
127 the ``config`` parameter which can be positional or use keyword form.
129 Useful attributes include:
131 - ``log``: an `logging.Logger` or subclass.
132 - ``config``: task-specific configuration; an instance of ``ConfigClass``
133 (see below).
134 - ``metadata``: a `TaskMetadata` for
135 collecting task-specific metadata, e.g. data quality and performance
136 metrics. This is data that is only meant to be persisted, never to be
137 used by the task.
139 Use a `lsst.pipe.base.PipelineTask` subclass to perform I/O with a
140 Butler.
142 Subclasses must also have an attribute ``ConfigClass`` that is a subclass
143 of `lsst.pex.config.Config` which configures the task. Subclasses should
144 also have an attribute ``_DefaultName``: the default name if there is no
145 parent task. ``_DefaultName`` is required for subclasses of
146 `~lsst.pipe.base.PipeLineTask` and recommended for subclasses of Task
147 because it simplifies construction (e.g. for unit tests).
148 """
150 ConfigClass: ClassVar[Type[Config]]
151 _DefaultName: ClassVar[str]
153 _add_module_logger_prefix: bool = True
154 """Control whether the module prefix should be prepended to default
155 logger names."""
157 def __init__(
158 self,
159 config: Optional[Config] = None,
160 *,
161 name: Optional[str] = None,
162 parentTask: Optional[Task] = None,
163 log: Optional[Union[logging.Logger, lsst.utils.logging.LsstLogAdapter]] = None,
164 ):
165 self.metadata = _TASK_METADATA_TYPE()
166 self.__parentTask: Optional[weakref.ReferenceType]
167 self.__parentTask = parentTask if parentTask is None else weakref.ref(parentTask)
169 if parentTask is not None:
170 if name is None:
171 raise RuntimeError("name is required for a subtask")
172 self._name = name
173 self._fullName = parentTask._computeFullName(name)
174 if config is None:
175 config = getattr(parentTask.config, name)
176 self._taskDict: Dict[str, weakref.ReferenceType[Task]] = parentTask._taskDict
177 loggerName = parentTask.log.getChild(name).name
178 else:
179 if name is None:
180 name = getattr(self, "_DefaultName", None)
181 if name is None:
182 raise RuntimeError("name is required for a task unless it has attribute _DefaultName")
183 name = self._DefaultName
184 self._name = name
185 self._fullName = self._name
186 if config is None:
187 config = self.ConfigClass()
188 self._taskDict = dict()
189 loggerName = self._fullName
190 if log is not None and log.name:
191 loggerName = log.getChild(loggerName).name
192 elif self._add_module_logger_prefix:
193 # Prefix the logger name with the root module name.
194 # We want all Task loggers to have this prefix to make
195 # it easier to control them. This can be disabled by
196 # a Task setting the class property _add_module_logger_prefix
197 # to False -- in which case the logger name will not be
198 # modified.
199 module_name = self.__module__
200 module_root = module_name.split(".")[0] + "."
201 if not loggerName.startswith(module_root):
202 loggerName = module_root + loggerName
204 # Get a logger (that might be a subclass of logging.Logger).
205 self.log: lsst.utils.logging.LsstLogAdapter = lsst.utils.logging.getLogger(loggerName)
206 self.config: Config = config
207 self.config.validate()
208 if lsstDebug:
209 self._display = lsstDebug.Info(self.__module__).display
210 else:
211 self._display = None
212 self._taskDict[self._fullName] = weakref.ref(self)
214 @property
215 def _parentTask(self) -> Optional[Task]:
216 return self.__parentTask if self.__parentTask is None else self.__parentTask()
218 def emptyMetadata(self) -> None:
219 """Empty (clear) the metadata for this Task and all sub-Tasks."""
220 for wref in self._taskDict.values():
221 subtask = wref()
222 assert subtask is not None, "Unexpected garbage collection of subtask."
223 subtask.metadata = _TASK_METADATA_TYPE()
225 def getFullMetadata(self) -> TaskMetadata:
226 """Get metadata for all tasks.
228 Returns
229 -------
230 metadata : `TaskMetadata`
231 The keys are the full task name.
232 Values are metadata for the top-level task and all subtasks,
233 sub-subtasks, etc.
235 Notes
236 -----
237 The returned metadata includes timing information (if
238 ``@timer.timeMethod`` is used) and any metadata set by the task. The
239 name of each item consists of the full task name with ``.`` replaced
240 by ``:``, followed by ``.`` and the name of the item, e.g.::
242 topLevelTaskName:subtaskName:subsubtaskName.itemName
244 using ``:`` in the full task name disambiguates the rare situation
245 that a task has a subtask and a metadata item with the same name.
246 """
247 fullMetadata = _TASK_FULL_METADATA_TYPE()
248 for fullName, wref in self.getTaskDict().items():
249 subtask = wref()
250 assert subtask is not None, "Unexpected garbage collection of subtask."
251 fullMetadata[fullName.replace(".", ":")] = subtask.metadata
252 return fullMetadata
254 def getFullName(self) -> str:
255 """Get the task name as a hierarchical name including parent task
256 names.
258 Returns
259 -------
260 fullName : `str`
261 The full name consists of the name of the parent task and each
262 subtask separated by periods. For example:
264 - The full name of top-level task "top" is simply "top".
265 - The full name of subtask "sub" of top-level task "top" is
266 "top.sub".
267 - The full name of subtask "sub2" of subtask "sub" of top-level
268 task "top" is "top.sub.sub2".
269 """
270 return self._fullName
272 def getName(self) -> str:
273 """Get the name of the task.
275 Returns
276 -------
277 taskName : `str`
278 Name of the task.
280 See also
281 --------
282 getFullName
283 """
284 return self._name
286 def getTaskDict(self) -> Dict[str, weakref.ReferenceType[Task]]:
287 """Get a dictionary of all tasks as a shallow copy.
289 Returns
290 -------
291 taskDict : `dict`
292 Dictionary containing full task name: task object for the top-level
293 task and all subtasks, sub-subtasks, etc.
294 """
295 return self._taskDict.copy()
297 def makeSubtask(self, name: str, **keyArgs: Any) -> None:
298 """Create a subtask as a new instance as the ``name`` attribute of this
299 task.
301 Parameters
302 ----------
303 name : `str`
304 Brief name of the subtask.
305 keyArgs
306 Extra keyword arguments used to construct the task. The following
307 arguments are automatically provided and cannot be overridden:
309 - "config".
310 - "parentTask".
312 Notes
313 -----
314 The subtask must be defined by ``Task.config.name``, an instance of
315 `~lsst.pex.config.ConfigurableField` or
316 `~lsst.pex.config.RegistryField`.
317 """
318 taskField = getattr(self.config, name, None)
319 if taskField is None:
320 raise KeyError(f"{self.getFullName()}'s config does not have field {name!r}")
321 subtask = taskField.apply(name=name, parentTask=self, **keyArgs)
322 setattr(self, name, subtask)
324 @contextlib.contextmanager
325 def timer(self, name: str, logLevel: int = logging.DEBUG) -> Iterator[None]:
326 """Context manager to log performance data for an arbitrary block of
327 code.
329 Parameters
330 ----------
331 name : `str`
332 Name of code being timed; data will be logged using item name:
333 ``Start`` and ``End``.
334 logLevel
335 A `logging` level constant.
337 Examples
338 --------
339 Creating a timer context:
341 .. code-block:: python
343 with self.timer("someCodeToTime"):
344 pass # code to time
346 See also
347 --------
348 timer.logInfo
349 """
350 logInfo(obj=self, prefix=name + "Start", logLevel=logLevel)
351 try:
352 yield
353 finally:
354 logInfo(obj=self, prefix=name + "End", logLevel=logLevel)
356 @classmethod
357 def makeField(cls, doc: str) -> ConfigurableField:
358 """Make a `lsst.pex.config.ConfigurableField` for this task.
360 Parameters
361 ----------
362 doc : `str`
363 Help text for the field.
365 Returns
366 -------
367 configurableField : `lsst.pex.config.ConfigurableField`
368 A `~ConfigurableField` for this task.
370 Examples
371 --------
372 Provides a convenient way to specify this task is a subtask of another
373 task.
375 Here is an example of use:
377 .. code-block:: python
379 class OtherTaskConfig(lsst.pex.config.Config):
380 aSubtask = ATaskClass.makeField("brief description of task")
381 """
382 return ConfigurableField(doc=doc, target=cls)
384 def _computeFullName(self, name: str) -> str:
385 """Compute the full name of a subtask or metadata item, given its brief
386 name.
388 Parameters
389 ----------
390 name : `str`
391 Brief name of subtask or metadata item.
393 Returns
394 -------
395 fullName : `str`
396 The full name: the ``name`` argument prefixed by the full task name
397 and a period.
399 Notes
400 -----
401 For example: if the full name of this task is "top.sub.sub2"
402 then ``_computeFullName("subname")`` returns
403 ``"top.sub.sub2.subname"``.
404 """
405 return f"{self._fullName}.{name}"
407 @staticmethod
408 def _unpickle_via_factory(
409 factory: Callable[..., Task], args: Sequence[Any], kwargs: Dict[str, Any]
410 ) -> Task:
411 """Unpickle something by calling a factory
413 Allows subclasses to unpickle using `__reduce__` with keyword
414 arguments as well as positional arguments.
415 """
416 return factory(*args, **kwargs)
418 def _reduce_kwargs(self) -> Dict[str, Any]:
419 """Returns a dict of the keyword arguments that should be used
420 by `__reduce__`.
422 Subclasses with additional arguments should always call the parent
423 class method to ensure that the standard parameters are included.
425 Returns
426 -------
427 kwargs : `dict`
428 Keyword arguments to be used when pickling.
429 """
430 return dict(
431 config=self.config,
432 name=self._name,
433 parentTask=self._parentTask,
434 )
436 def __reduce__(
437 self,
438 ) -> Tuple[
439 Callable[[Callable[..., Task], Sequence[Any], Dict[str, Any]], Task],
440 Tuple[Type[Task], Sequence[Any], Dict[str, Any]],
441 ]:
442 """Pickler."""
443 return self._unpickle_via_factory, (self.__class__, [], self._reduce_kwargs())