Coverage for python/lsst/pipe/base/graph/quantumNode.py: 61%

59 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-27 02:47 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ("QuantumNode", "NodeId", "BuildId") 

24 

25import uuid 

26import warnings 

27from dataclasses import dataclass 

28from typing import Any, Dict, NewType, Optional, Tuple 

29 

30from lsst.daf.butler import ( 

31 DatasetRef, 

32 DimensionRecord, 

33 DimensionRecordsAccumulator, 

34 DimensionUniverse, 

35 Quantum, 

36 SerializedQuantum, 

37 UnresolvedRefWarning, 

38) 

39from pydantic import BaseModel 

40 

41from ..pipeline import TaskDef 

42 

43BuildId = NewType("BuildId", str) 

44 

45 

46def _hashDsRef(ref: DatasetRef) -> int: 

47 return hash((ref.datasetType, ref.dataId)) 

48 

49 

50@dataclass(frozen=True, eq=True) 

51class NodeId: 

52 """Deprecated, this class is used with QuantumGraph save formats of 

53 1 and 2 when unpicking objects and must be retained until those formats 

54 are considered unloadable. 

55 

56 This represents an unique identifier of a node within an individual 

57 construction of a `QuantumGraph`. This identifier will stay constant 

58 through a pickle, and any `QuantumGraph` methods that return a new 

59 `QuantumGraph`. 

60 

61 A `NodeId` will not be the same if a new graph is built containing the same 

62 information in a `QuantumNode`, or even built from exactly the same inputs. 

63 

64 `NodeId`s do not play any role in deciding the equality or identity (hash) 

65 of a `QuantumNode`, and are mainly useful in debugging or working with 

66 various subsets of the same graph. 

67 

68 This interface is a convenance only, and no guarantees on long term 

69 stability are made. New implementations might change the `NodeId`, or 

70 provide more or less guarantees. 

71 """ 

72 

73 number: int 

74 """The unique position of the node within the graph assigned at graph 

75 creation. 

76 """ 

77 buildId: BuildId 

78 """Unique identifier created at the time the originating graph was created 

79 """ 

80 

81 

82@dataclass(frozen=True) 

83class QuantumNode: 

84 """This class represents a node in the quantum graph. 

85 

86 The quantum attribute represents the data that is to be processed at this 

87 node. 

88 """ 

89 

90 quantum: Quantum 

91 """The unit of data that is to be processed by this graph node""" 

92 taskDef: TaskDef 

93 """Definition of the task that will process the `Quantum` associated with 

94 this node. 

95 """ 

96 nodeId: uuid.UUID 

97 """The unique position of the node within the graph assigned at graph 

98 creation. 

99 """ 

100 

101 def __post_init__(self) -> None: 

102 # use setattr here to preserve the frozenness of the QuantumNode 

103 self._precomputedHash: int 

104 object.__setattr__(self, "_precomputedHash", hash((self.taskDef.label, self.quantum))) 

105 

106 def __eq__(self, other: object) -> bool: 

107 if not isinstance(other, QuantumNode): 

108 return False 

109 if self.quantum != other.quantum: 

110 return False 

111 return self.taskDef == other.taskDef 

112 

113 def __hash__(self) -> int: 

114 """For graphs it is useful to have a more robust hash than provided 

115 by the default quantum id based hashing 

116 """ 

117 return self._precomputedHash 

118 

119 def __repr__(self) -> str: 

120 """Make more human readable string representation.""" 

121 return ( 

122 f"{self.__class__.__name__}(quantum={self.quantum}, taskDef={self.taskDef}, nodeId={self.nodeId})" 

123 ) 

124 

125 def to_simple(self, accumulator: Optional[DimensionRecordsAccumulator] = None) -> SerializedQuantumNode: 

126 return SerializedQuantumNode( 

127 quantum=self.quantum.to_simple(accumulator=accumulator), 

128 taskLabel=self.taskDef.label, 

129 nodeId=self.nodeId, 

130 ) 

131 

132 @classmethod 

133 def from_simple( 

134 cls, 

135 simple: SerializedQuantumNode, 

136 taskDefMap: Dict[str, TaskDef], 

137 universe: DimensionUniverse, 

138 recontitutedDimensions: Optional[Dict[int, Tuple[str, DimensionRecord]]] = None, 

139 ) -> QuantumNode: 

140 with warnings.catch_warnings(): 

141 warnings.simplefilter("ignore", category=UnresolvedRefWarning) 

142 return QuantumNode( 

143 quantum=Quantum.from_simple( 

144 simple.quantum, universe, reconstitutedDimensions=recontitutedDimensions 

145 ), 

146 taskDef=taskDefMap[simple.taskLabel], 

147 nodeId=simple.nodeId, 

148 ) 

149 

150 

151class SerializedQuantumNode(BaseModel): 

152 quantum: SerializedQuantum 

153 taskLabel: str 

154 nodeId: uuid.UUID 

155 

156 @classmethod 

157 def direct(cls, *, quantum: Dict[str, Any], taskLabel: str, nodeId: str) -> SerializedQuantumNode: 

158 node = SerializedQuantumNode.__new__(cls) 

159 setter = object.__setattr__ 

160 setter(node, "quantum", SerializedQuantum.direct(**quantum)) 

161 setter(node, "taskLabel", taskLabel) 

162 setter(node, "nodeId", uuid.UUID(nodeId)) 

163 setter(node, "__fields_set__", {"quantum", "taskLabel", "nodeId"}) 

164 return node