Coverage for python / lsst / pipe / base / quantum_graph / aggregator / _structs.py: 99%

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ( 

31 "InProgressScan", 

32 "IngestRequest", 

33 "ScanReport", 

34 "ScanStatus", 

35 "WriteRequest", 

36) 

37 

38import dataclasses 

39import enum 

40import uuid 

41 

42from lsst.daf.butler.datastore.record_data import DatastoreRecordData 

43 

44from .._common import DatastoreName 

45from .._predicted import PredictedDatasetModel 

46from .._provenance import ( 

47 ProvenanceLogRecordsModel, 

48 ProvenanceQuantumAttemptModel, 

49 ProvenanceTaskMetadataModel, 

50) 

51 

52 

53class ScanStatus(enum.Enum): 

54 """Status enum for quantum scanning. 

55 

56 Note that this records the status for the *scanning* which is distinct 

57 from the status of the quantum's execution. 

58 """ 

59 

60 INCOMPLETE = enum.auto() 

61 """The quantum is not necessarily done running, and cannot be scanned 

62 conclusively yet. 

63 """ 

64 

65 ABANDONED = enum.auto() 

66 """The quantum's execution appears to have failed but we cannot rule out 

67 the possibility that it could be recovered, but we've also waited long 

68 enough (according to `ScannerTimeConfigDict.retry_timeout`) that it's time 

69 to stop trying for now. 

70 

71 This state means a later run with `ScannerConfig.assume_complete` is 

72 required. 

73 """ 

74 

75 SUCCESSFUL = enum.auto() 

76 """The quantum was conclusively scanned and was executed successfully, 

77 unblocking scans for downstream quanta. 

78 """ 

79 

80 FAILED = enum.auto() 

81 """The quantum was conclusively scanned and failed execution, blocking 

82 scans for downstream quanta. 

83 """ 

84 

85 BLOCKED = enum.auto() 

86 """A quantum upstream of this one failed.""" 

87 

88 INIT = enum.auto() 

89 """Init quanta need special handling, because they don't have logs and 

90 metadata. 

91 """ 

92 

93 

94@dataclasses.dataclass 

95class ScanReport: 

96 """Minimal information needed about a completed scan by the supervisor.""" 

97 

98 quantum_id: uuid.UUID 

99 """Unique ID of the quantum.""" 

100 

101 status: ScanStatus 

102 """Combined status of the scan and the execution of the quantum.""" 

103 

104 

105@dataclasses.dataclass 

106class IngestRequest: 

107 """A request to ingest datasets produced by a single quantum.""" 

108 

109 producer_id: uuid.UUID 

110 """ID of the quantum that produced these datasets.""" 

111 

112 datasets: list[PredictedDatasetModel] 

113 """Registry information about the datasets.""" 

114 

115 records: dict[DatastoreName, DatastoreRecordData] 

116 """Datastore information about the datasets.""" 

117 

118 def __bool__(self) -> bool: 

119 return bool(self.datasets or self.records) 

120 

121 

122@dataclasses.dataclass 

123class InProgressScan: 

124 """A struct that represents a quantum that is being scanned.""" 

125 

126 quantum_id: uuid.UUID 

127 """Unique ID for the quantum.""" 

128 

129 status: ScanStatus 

130 """Combined status for the scan and the execution of the quantum.""" 

131 

132 attempts: list[ProvenanceQuantumAttemptModel] = dataclasses.field(default_factory=list) 

133 """Provenance information about each attempt to run the quantum.""" 

134 

135 outputs: dict[uuid.UUID, bool] = dataclasses.field(default_factory=dict) 

136 """Unique IDs of the output datasets mapped to whether they were actually 

137 produced. 

138 """ 

139 

140 metadata: ProvenanceTaskMetadataModel = dataclasses.field(default_factory=ProvenanceTaskMetadataModel) 

141 """Task metadata information for each attempt. 

142 """ 

143 

144 logs: ProvenanceLogRecordsModel = dataclasses.field(default_factory=ProvenanceLogRecordsModel) 

145 """Log records for each attempt. 

146 """ 

147 

148 

149@dataclasses.dataclass 

150class WriteRequest: 

151 """A struct that represents a request to write provenance for a quantum.""" 

152 

153 quantum_id: uuid.UUID 

154 """Unique ID for the quantum.""" 

155 

156 status: ScanStatus 

157 """Combined status for the scan and the execution of the quantum.""" 

158 

159 existing_outputs: set[uuid.UUID] = dataclasses.field(default_factory=set) 

160 """Unique IDs of the output datasets that were actually written.""" 

161 

162 quantum: bytes = b"" 

163 """Serialized quantum provenance model. 

164 

165 This may be empty for quanta that had no attempts. 

166 """ 

167 

168 metadata: bytes = b"" 

169 """Serialized task metadata.""" 

170 

171 logs: bytes = b"" 

172 """Serialized logs.""" 

173 

174 is_compressed: bool = False 

175 """Whether the `quantum`, `metadata`, and `log` attributes are 

176 compressed. 

177 """