Coverage for python/lsst/pipe/base/_status.py: 67%

31 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-27 02:40 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30import abc 

31import logging 

32from typing import Protocol 

33 

34from lsst.utils import introspection 

35 

36from ._task_metadata import GetSetDictMetadata, NestedMetadataDict 

37 

38__all__ = ( 

39 "UnprocessableDataError", 

40 "AnnotatedPartialOutputsError", 

41 "NoWorkFound", 

42 "RepeatableQuantumError", 

43 "AlgorithmError", 

44 "InvalidQuantumError", 

45) 

46 

47 

48class GetSetDictMetadataHolder(Protocol): 

49 """Protocol for objects that have a ``metadata`` attribute that satisfies 

50 `GetSetDictMetadata`. 

51 """ 

52 

53 metadata: GetSetDictMetadata | None 

54 

55 

56class NoWorkFound(BaseException): 

57 """An exception raised when a Quantum should not exist because there is no 

58 work for it to do. 

59 

60 This usually occurs because a non-optional input dataset is not present, or 

61 a spatiotemporal overlap that was conservatively predicted does not 

62 actually exist. 

63 

64 This inherits from BaseException because it is used to signal a case that 

65 we don't consider a real error, even though we often want to use try/except 

66 logic to trap it. 

67 """ 

68 

69 

70class RepeatableQuantumError(RuntimeError): 

71 """Exception that may be raised by PipelineTasks (and code they delegate 

72 to) in order to indicate that a repeatable problem that will not be 

73 addressed by retries. 

74 

75 This usually indicates that the algorithm and the data it has been given 

76 are somehow incompatible, and the task should run fine on most other data. 

77 

78 This exception may be used as a base class for more specific questions, or 

79 used directly while chaining another exception, e.g.:: 

80 

81 try: 

82 run_code() 

83 except SomeOtherError as err: 

84 raise RepeatableQuantumError() from err 

85 

86 This may be used for missing input data when the desired behavior is to 

87 cause all downstream tasks being run be blocked, forcing the user to 

88 address the problem. When the desired behavior is to skip all of this 

89 quantum and attempt downstream tasks (or skip them) without its its 

90 outputs, raise `NoWorkFound` or return without raising instead. 

91 """ 

92 

93 EXIT_CODE = 20 

94 

95 

96class AlgorithmError(RepeatableQuantumError, abc.ABC): 

97 """Exception that may be raised by PipelineTasks (and code they delegate 

98 to) in order to indicate a repeatable algorithmic failure that will not be 

99 addressed by retries. 

100 

101 Subclass this exception to define the metadata associated with the error 

102 (for example: number of data points in a fit vs. degrees of freedom). 

103 """ 

104 

105 @property 

106 @abc.abstractmethod 

107 def metadata(self) -> NestedMetadataDict | None: 

108 """Metadata from the raising `~lsst.pipe.base.Task` with more 

109 information about the failure. The contents of the dict are 

110 `~lsst.pipe.base.Task`-dependent, and must have `str` keys and `str`, 

111 `int`, `float`, `bool`, or nested-dictionary (with the same key and 

112 value types) values. 

113 """ 

114 raise NotImplementedError 

115 

116 

117class UnprocessableDataError(RepeatableQuantumError): 

118 """Exception that will be subclassed and raised by Tasks to indicate a 

119 failure to process their inputs for some reason that is non-recoverable. 

120 

121 Notes 

122 ----- 

123 An example is a known bright star that causes PSF measurement to fail, and 

124 that makes that detector entirely non-recoverable. 

125 

126 Do not raise this unless we are convinced that the data cannot be 

127 processed, even by a better algorithm. Most instances where this error 

128 would be raised likely require an RFC to explicitly define the situation. 

129 """ 

130 

131 

132class AnnotatedPartialOutputsError(RepeatableQuantumError): 

133 """Exception that runQuantum raises when the (partial) outputs it has 

134 written contain information about their own incompleteness or degraded 

135 quality. 

136 

137 This exception should always chain the original error. When the 

138 executor catches this exception, it will report the original exception. In 

139 contrast, other exceptions raised from ``runQuantum`` are considered to 

140 invalidate any outputs that are already written. 

141 """ 

142 

143 @classmethod 

144 def annotate( 

145 cls, error: Exception, *args: GetSetDictMetadataHolder | None, log: logging.Logger 

146 ) -> AnnotatedPartialOutputsError: 

147 """Set metadata on outputs to explain the nature of the failure. 

148 

149 Parameters 

150 ---------- 

151 error : `Exception` 

152 Exception that caused the task to fail. 

153 *args : `GetSetDictMetadataHolder` 

154 Objects (e.g. Task, Exposure, SimpleCatalog) to annotate with 

155 failure information. They must have a `metadata` property. 

156 log : `logging.Logger` 

157 Log to send error message to. 

158 

159 Returns 

160 ------- 

161 error : `AnnotatedPartialOutputsError` 

162 Exception that the failing task can ``raise from`` with the 

163 passed-in exception. 

164 

165 Notes 

166 ----- 

167 This should be called from within an except block that has caught an 

168 exception. Here is an example of handling a failure in 

169 ``PipelineTask.runQuantum`` that annotates and writes partial outputs: 

170 

171 .. code-block:: py 

172 :name: annotate-error-example 

173 

174 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

175 inputs = butlerQC.get(inputRefs) 

176 exposures = inputs.pop("exposures") 

177 assert not inputs, "runQuantum got more inputs than expected" 

178 

179 result = pipeBase.Struct(catalog=None) 

180 try: 

181 self.run(exposure) 

182 except pipeBase.AlgorithmError as e: 

183 error = pipeBase.AnnotatedPartialOutputsError.annotate( 

184 e, self, result.catalog, log=self.log 

185 ) 

186 raise error from e 

187 finally: 

188 butlerQC.put(result, outputRefs) 

189 """ 

190 failure_info = { 

191 "message": str(error), 

192 "type": introspection.get_full_type_name(error), 

193 } 

194 if other := getattr(error, "metadata", None): 

195 failure_info["metadata"] = other 

196 

197 # NOTE: Can't fully test this in pipe_base because afw is not a 

198 # dependency; test_calibrateImage.py in pipe_tasks gives more coverage. 

199 for item in args: 

200 # Some outputs may not exist, so we cannot set metadata on them. 

201 if item is None: 

202 continue 

203 item.metadata.set_dict("failure", failure_info) # type: ignore 

204 

205 log.exception( 

206 "Task failed with only partial outputs; see exception message for details.", 

207 exc_info=error, 

208 ) 

209 

210 return cls("Task failed and wrote partial outputs: see chained exception for details.") 

211 

212 

213class InvalidQuantumError(Exception): 

214 """Exception that may be raised by PipelineTasks (and code they delegate 

215 to) in order to indicate logic bug or configuration problem. 

216 

217 This usually indicates that the configured algorithm itself is invalid and 

218 will not run on a significant fraction of quanta (often all of them). 

219 

220 This exception may be used as a base class for more specific questions, or 

221 used directly while chaining another exception, e.g.:: 

222 

223 try: 

224 run_code() 

225 except SomeOtherError as err: 

226 raise RepeatableQuantumError() from err 

227 

228 Raising this exception in `PipelineTask.runQuantum` or something it calls 

229 is a last resort - whenever possible, such problems should cause exceptions 

230 in ``__init__`` or in QuantumGraph generation. It should never be used 

231 for missing data. 

232 """ 

233 

234 EXIT_CODE = 21