Coverage for python/lsst/pipe/base/graph/quantumNode.py: 65%

61 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-23 10:31 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ("QuantumNode", "NodeId", "BuildId") 

24 

25import uuid 

26import warnings 

27from dataclasses import dataclass 

28from typing import Any, NewType 

29 

30from lsst.daf.butler import ( 

31 DatasetRef, 

32 DimensionRecord, 

33 DimensionRecordsAccumulator, 

34 DimensionUniverse, 

35 Quantum, 

36 SerializedQuantum, 

37) 

38from lsst.daf.butler._compat import _BaseModelCompat 

39from lsst.utils.introspection import find_outside_stacklevel 

40 

41from ..pipeline import TaskDef 

42 

43BuildId = NewType("BuildId", str) 

44 

45 

46def _hashDsRef(ref: DatasetRef) -> int: 

47 return hash((ref.datasetType, ref.dataId)) 

48 

49 

50@dataclass(frozen=True, eq=True) 

51class NodeId: 

52 """Deprecated, this class is used with QuantumGraph save formats of 

53 1 and 2 when unpicking objects and must be retained until those formats 

54 are considered unloadable. 

55 

56 This represents an unique identifier of a node within an individual 

57 construction of a `QuantumGraph`. This identifier will stay constant 

58 through a pickle, and any `QuantumGraph` methods that return a new 

59 `QuantumGraph`. 

60 

61 A `NodeId` will not be the same if a new graph is built containing the same 

62 information in a `QuantumNode`, or even built from exactly the same inputs. 

63 

64 `NodeId`s do not play any role in deciding the equality or identity (hash) 

65 of a `QuantumNode`, and are mainly useful in debugging or working with 

66 various subsets of the same graph. 

67 

68 This interface is a convenance only, and no guarantees on long term 

69 stability are made. New implementations might change the `NodeId`, or 

70 provide more or less guarantees. 

71 """ 

72 

73 number: int 

74 """The unique position of the node within the graph assigned at graph 

75 creation. 

76 """ 

77 buildId: BuildId 

78 """Unique identifier created at the time the originating graph was created 

79 """ 

80 

81 

82@dataclass(frozen=True) 

83class QuantumNode: 

84 """Class representing a node in the quantum graph. 

85 

86 The ``quantum`` attribute represents the data that is to be processed at 

87 this node. 

88 """ 

89 

90 quantum: Quantum 

91 """The unit of data that is to be processed by this graph node""" 

92 taskDef: TaskDef 

93 """Definition of the task that will process the `Quantum` associated with 

94 this node. 

95 """ 

96 nodeId: uuid.UUID 

97 """The unique position of the node within the graph assigned at graph 

98 creation. 

99 """ 

100 

101 __slots__ = ("quantum", "taskDef", "nodeId", "_precomputedHash") 

102 

103 def __post_init__(self) -> None: 

104 # use setattr here to preserve the frozenness of the QuantumNode 

105 self._precomputedHash: int 

106 object.__setattr__(self, "_precomputedHash", hash((self.taskDef.label, self.quantum))) 

107 

108 def __eq__(self, other: object) -> bool: 

109 if not isinstance(other, QuantumNode): 

110 return False 

111 if self.quantum != other.quantum: 

112 return False 

113 return self.taskDef == other.taskDef 

114 

115 def __hash__(self) -> int: 

116 """For graphs it is useful to have a more robust hash than provided 

117 by the default quantum id based hashing 

118 """ 

119 return self._precomputedHash 

120 

121 def __repr__(self) -> str: 

122 """Make more human readable string representation.""" 

123 return ( 

124 f"{self.__class__.__name__}(quantum={self.quantum}, taskDef={self.taskDef}, nodeId={self.nodeId})" 

125 ) 

126 

127 def to_simple(self, accumulator: DimensionRecordsAccumulator | None = None) -> SerializedQuantumNode: 

128 return SerializedQuantumNode( 

129 quantum=self.quantum.to_simple(accumulator=accumulator), 

130 taskLabel=self.taskDef.label, 

131 nodeId=self.nodeId, 

132 ) 

133 

134 @classmethod 

135 def from_simple( 

136 cls, 

137 simple: SerializedQuantumNode, 

138 taskDefMap: dict[str, TaskDef], 

139 universe: DimensionUniverse, 

140 recontitutedDimensions: dict[int, tuple[str, DimensionRecord]] | None = None, 

141 ) -> QuantumNode: 

142 if recontitutedDimensions is not None: 

143 warnings.warn( 

144 "The recontitutedDimensions argument is now ignored and may be removed after v26", 

145 category=FutureWarning, 

146 stacklevel=find_outside_stacklevel("lsst.pipe.base"), 

147 ) 

148 return QuantumNode( 

149 quantum=Quantum.from_simple(simple.quantum, universe), 

150 taskDef=taskDefMap[simple.taskLabel], 

151 nodeId=simple.nodeId, 

152 ) 

153 

154 def _replace_quantum(self, quantum: Quantum) -> None: 

155 """Replace Quantum instance in this node. 

156 

157 Parameters 

158 ---------- 

159 quantum : `Quantum` 

160 New Quantum instance for this node. 

161 

162 Raises 

163 ------ 

164 ValueError 

165 Raised if the hash of the new quantum is different from the hash of 

166 the existing quantum. 

167 

168 Notes 

169 ----- 

170 This class is immutable and hashable, so this method checks that new 

171 quantum does not invalidate its current hash. This method is supposed 

172 to used only by `QuantumGraph` class as its implementation detail, 

173 so it is made "underscore-protected". 

174 """ 

175 if hash(quantum) != hash(self.quantum): 

176 raise ValueError( 

177 f"Hash of the new quantum {quantum} does not match hash of existing quantum {self.quantum}" 

178 ) 

179 object.__setattr__(self, "quantum", quantum) 

180 

181 

182_fields_set = {"quantum", "taskLabel", "nodeId"} 

183 

184 

185class SerializedQuantumNode(_BaseModelCompat): 

186 """Model representing a `QuantumNode` in serializable form.""" 

187 

188 quantum: SerializedQuantum 

189 taskLabel: str 

190 nodeId: uuid.UUID 

191 

192 @classmethod 

193 def direct(cls, *, quantum: dict[str, Any], taskLabel: str, nodeId: str) -> SerializedQuantumNode: 

194 node = cls.model_construct( 

195 __fields_set=_fields_set, 

196 quantum=SerializedQuantum.direct(**quantum), 

197 taskLabel=taskLabel, 

198 nodeId=uuid.UUID(nodeId), 

199 ) 

200 

201 return node