Coverage for python/lsst/pipe/base/graph/quantumNode.py: 65%

61 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-19 10:39 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ("QuantumNode", "NodeId", "BuildId") 

30 

31import uuid 

32import warnings 

33from dataclasses import dataclass 

34from typing import Any, NewType 

35 

36from lsst.daf.butler import ( 

37 DatasetRef, 

38 DimensionRecord, 

39 DimensionRecordsAccumulator, 

40 DimensionUniverse, 

41 Quantum, 

42 SerializedQuantum, 

43) 

44from lsst.daf.butler._compat import _BaseModelCompat 

45from lsst.utils.introspection import find_outside_stacklevel 

46 

47from ..pipeline import TaskDef 

48 

49BuildId = NewType("BuildId", str) 

50 

51 

52def _hashDsRef(ref: DatasetRef) -> int: 

53 return hash((ref.datasetType, ref.dataId)) 

54 

55 

56@dataclass(frozen=True, eq=True) 

57class NodeId: 

58 """Deprecated, this class is used with QuantumGraph save formats of 

59 1 and 2 when unpicking objects and must be retained until those formats 

60 are considered unloadable. 

61 

62 This represents an unique identifier of a node within an individual 

63 construction of a `QuantumGraph`. This identifier will stay constant 

64 through a pickle, and any `QuantumGraph` methods that return a new 

65 `QuantumGraph`. 

66 

67 A `NodeId` will not be the same if a new graph is built containing the same 

68 information in a `QuantumNode`, or even built from exactly the same inputs. 

69 

70 `NodeId`s do not play any role in deciding the equality or identity (hash) 

71 of a `QuantumNode`, and are mainly useful in debugging or working with 

72 various subsets of the same graph. 

73 

74 This interface is a convenance only, and no guarantees on long term 

75 stability are made. New implementations might change the `NodeId`, or 

76 provide more or less guarantees. 

77 """ 

78 

79 number: int 

80 """The unique position of the node within the graph assigned at graph 

81 creation. 

82 """ 

83 buildId: BuildId 

84 """Unique identifier created at the time the originating graph was created 

85 """ 

86 

87 

88@dataclass(frozen=True) 

89class QuantumNode: 

90 """Class representing a node in the quantum graph. 

91 

92 The ``quantum`` attribute represents the data that is to be processed at 

93 this node. 

94 """ 

95 

96 quantum: Quantum 

97 """The unit of data that is to be processed by this graph node""" 

98 taskDef: TaskDef 

99 """Definition of the task that will process the `Quantum` associated with 

100 this node. 

101 """ 

102 nodeId: uuid.UUID 

103 """The unique position of the node within the graph assigned at graph 

104 creation. 

105 """ 

106 

107 __slots__ = ("quantum", "taskDef", "nodeId", "_precomputedHash") 

108 

109 def __post_init__(self) -> None: 

110 # use setattr here to preserve the frozenness of the QuantumNode 

111 self._precomputedHash: int 

112 object.__setattr__(self, "_precomputedHash", hash((self.taskDef.label, self.quantum))) 

113 

114 def __eq__(self, other: object) -> bool: 

115 if not isinstance(other, QuantumNode): 

116 return False 

117 if self.quantum != other.quantum: 

118 return False 

119 return self.taskDef == other.taskDef 

120 

121 def __hash__(self) -> int: 

122 """For graphs it is useful to have a more robust hash than provided 

123 by the default quantum id based hashing 

124 """ 

125 return self._precomputedHash 

126 

127 def __repr__(self) -> str: 

128 """Make more human readable string representation.""" 

129 return ( 

130 f"{self.__class__.__name__}(quantum={self.quantum}, taskDef={self.taskDef}, nodeId={self.nodeId})" 

131 ) 

132 

133 def to_simple(self, accumulator: DimensionRecordsAccumulator | None = None) -> SerializedQuantumNode: 

134 return SerializedQuantumNode( 

135 quantum=self.quantum.to_simple(accumulator=accumulator), 

136 taskLabel=self.taskDef.label, 

137 nodeId=self.nodeId, 

138 ) 

139 

140 @classmethod 

141 def from_simple( 

142 cls, 

143 simple: SerializedQuantumNode, 

144 taskDefMap: dict[str, TaskDef], 

145 universe: DimensionUniverse, 

146 recontitutedDimensions: dict[int, tuple[str, DimensionRecord]] | None = None, 

147 ) -> QuantumNode: 

148 if recontitutedDimensions is not None: 

149 warnings.warn( 

150 "The recontitutedDimensions argument is now ignored and may be removed after v26", 

151 category=FutureWarning, 

152 stacklevel=find_outside_stacklevel("lsst.pipe.base"), 

153 ) 

154 return QuantumNode( 

155 quantum=Quantum.from_simple(simple.quantum, universe), 

156 taskDef=taskDefMap[simple.taskLabel], 

157 nodeId=simple.nodeId, 

158 ) 

159 

160 def _replace_quantum(self, quantum: Quantum) -> None: 

161 """Replace Quantum instance in this node. 

162 

163 Parameters 

164 ---------- 

165 quantum : `Quantum` 

166 New Quantum instance for this node. 

167 

168 Raises 

169 ------ 

170 ValueError 

171 Raised if the hash of the new quantum is different from the hash of 

172 the existing quantum. 

173 

174 Notes 

175 ----- 

176 This class is immutable and hashable, so this method checks that new 

177 quantum does not invalidate its current hash. This method is supposed 

178 to used only by `QuantumGraph` class as its implementation detail, 

179 so it is made "underscore-protected". 

180 """ 

181 if hash(quantum) != hash(self.quantum): 

182 raise ValueError( 

183 f"Hash of the new quantum {quantum} does not match hash of existing quantum {self.quantum}" 

184 ) 

185 object.__setattr__(self, "quantum", quantum) 

186 

187 

188_fields_set = {"quantum", "taskLabel", "nodeId"} 

189 

190 

191class SerializedQuantumNode(_BaseModelCompat): 

192 """Model representing a `QuantumNode` in serializable form.""" 

193 

194 quantum: SerializedQuantum 

195 taskLabel: str 

196 nodeId: uuid.UUID 

197 

198 @classmethod 

199 def direct(cls, *, quantum: dict[str, Any], taskLabel: str, nodeId: str) -> SerializedQuantumNode: 

200 node = cls.model_construct( 

201 __fields_set=_fields_set, 

202 quantum=SerializedQuantum.direct(**quantum), 

203 taskLabel=taskLabel, 

204 nodeId=uuid.UUID(nodeId), 

205 ) 

206 

207 return node