Coverage for python/lsst/pipe/base/_status.py: 67%
31 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:55 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:55 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30import abc
31import logging
32from typing import Protocol
34from lsst.utils import introspection
36from ._task_metadata import GetSetDictMetadata, NestedMetadataDict
38__all__ = (
39 "UnprocessableDataError",
40 "AnnotatedPartialOutputsError",
41 "NoWorkFound",
42 "RepeatableQuantumError",
43 "AlgorithmError",
44 "InvalidQuantumError",
45)
48class GetSetDictMetadataHolder(Protocol):
49 """Protocol for objects that have a ``metadata`` attribute that satisfies
50 `GetSetDictMetadata`.
51 """
53 metadata: GetSetDictMetadata | None
56class NoWorkFound(BaseException):
57 """An exception raised when a Quantum should not exist because there is no
58 work for it to do.
60 This usually occurs because a non-optional input dataset is not present, or
61 a spatiotemporal overlap that was conservatively predicted does not
62 actually exist.
64 This inherits from BaseException because it is used to signal a case that
65 we don't consider a real error, even though we often want to use try/except
66 logic to trap it.
67 """
70class RepeatableQuantumError(RuntimeError):
71 """Exception that may be raised by PipelineTasks (and code they delegate
72 to) in order to indicate that a repeatable problem that will not be
73 addressed by retries.
75 This usually indicates that the algorithm and the data it has been given
76 are somehow incompatible, and the task should run fine on most other data.
78 This exception may be used as a base class for more specific questions, or
79 used directly while chaining another exception, e.g.::
81 try:
82 run_code()
83 except SomeOtherError as err:
84 raise RepeatableQuantumError() from err
86 This may be used for missing input data when the desired behavior is to
87 cause all downstream tasks being run be blocked, forcing the user to
88 address the problem. When the desired behavior is to skip all of this
89 quantum and attempt downstream tasks (or skip them) without its its
90 outputs, raise `NoWorkFound` or return without raising instead.
91 """
93 EXIT_CODE = 20
96class AlgorithmError(RepeatableQuantumError, abc.ABC):
97 """Exception that may be raised by PipelineTasks (and code they delegate
98 to) in order to indicate a repeatable algorithmic failure that will not be
99 addressed by retries.
101 Subclass this exception to define the metadata associated with the error
102 (for example: number of data points in a fit vs. degrees of freedom).
103 """
105 @property
106 @abc.abstractmethod
107 def metadata(self) -> NestedMetadataDict | None:
108 """Metadata from the raising `~lsst.pipe.base.Task` with more
109 information about the failure. The contents of the dict are
110 `~lsst.pipe.base.Task`-dependent, and must have `str` keys and `str`,
111 `int`, `float`, `bool`, or nested-dictionary (with the same key and
112 value types) values.
113 """
114 raise NotImplementedError
117class UnprocessableDataError(RepeatableQuantumError):
118 """Exception that will be subclassed and raised by Tasks to indicate a
119 failure to process their inputs for some reason that is non-recoverable.
121 Notes
122 -----
123 An example is a known bright star that causes PSF measurement to fail, and
124 that makes that detector entirely non-recoverable.
126 Do not raise this unless we are convinced that the data cannot be
127 processed, even by a better algorithm. Most instances where this error
128 would be raised likely require an RFC to explicitly define the situation.
129 """
132class AnnotatedPartialOutputsError(RepeatableQuantumError):
133 """Exception that runQuantum raises when the (partial) outputs it has
134 written contain information about their own incompleteness or degraded
135 quality.
137 This exception should always chain the original error. When the
138 executor catches this exception, it will report the original exception. In
139 contrast, other exceptions raised from ``runQuantum`` are considered to
140 invalidate any outputs that are already written.
141 """
143 @classmethod
144 def annotate(
145 cls, error: Exception, *args: GetSetDictMetadataHolder | None, log: logging.Logger
146 ) -> AnnotatedPartialOutputsError:
147 """Set metadata on outputs to explain the nature of the failure.
149 Parameters
150 ----------
151 error : `Exception`
152 Exception that caused the task to fail.
153 *args : `GetSetDictMetadataHolder`
154 Objects (e.g. Task, Exposure, SimpleCatalog) to annotate with
155 failure information. They must have a `metadata` property.
156 log : `logging.Logger`
157 Log to send error message to.
159 Returns
160 -------
161 error : `AnnotatedPartialOutputsError`
162 Exception that the failing task can ``raise from`` with the
163 passed-in exception.
165 Notes
166 -----
167 This should be called from within an except block that has caught an
168 exception. Here is an example of handling a failure in
169 ``PipelineTask.runQuantum`` that annotates and writes partial outputs:
171 .. code-block:: py
172 :name: annotate-error-example
174 def runQuantum(self, butlerQC, inputRefs, outputRefs):
175 inputs = butlerQC.get(inputRefs)
176 exposures = inputs.pop("exposures")
177 assert not inputs, "runQuantum got more inputs than expected"
179 result = pipeBase.Struct(catalog=None)
180 try:
181 self.run(exposure)
182 except pipeBase.AlgorithmError as e:
183 error = pipeBase.AnnotatedPartialOutputsError.annotate(
184 e, self, result.catalog, log=self.log
185 )
186 raise error from e
187 finally:
188 butlerQC.put(result, outputRefs)
189 """
190 failure_info = {
191 "message": str(error),
192 "type": introspection.get_full_type_name(error),
193 }
194 if other := getattr(error, "metadata", None):
195 failure_info["metadata"] = other
197 # NOTE: Can't fully test this in pipe_base because afw is not a
198 # dependency; test_calibrateImage.py in pipe_tasks gives more coverage.
199 for item in args:
200 # Some outputs may not exist, so we cannot set metadata on them.
201 if item is None:
202 continue
203 item.metadata.set_dict("failure", failure_info) # type: ignore
205 log.exception(
206 "Task failed with only partial outputs; see exception message for details.",
207 exc_info=error,
208 )
210 return cls("Task failed and wrote partial outputs: see chained exception for details.")
213class InvalidQuantumError(Exception):
214 """Exception that may be raised by PipelineTasks (and code they delegate
215 to) in order to indicate logic bug or configuration problem.
217 This usually indicates that the configured algorithm itself is invalid and
218 will not run on a significant fraction of quanta (often all of them).
220 This exception may be used as a base class for more specific questions, or
221 used directly while chaining another exception, e.g.::
223 try:
224 run_code()
225 except SomeOtherError as err:
226 raise RepeatableQuantumError() from err
228 Raising this exception in `PipelineTask.runQuantum` or something it calls
229 is a last resort - whenever possible, such problems should cause exceptions
230 in ``__init__`` or in QuantumGraph generation. It should never be used
231 for missing data.
232 """
234 EXIT_CODE = 21