Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 ClassVar, 

30 Dict, 

31 FrozenSet, 

32 Iterable, 

33 List, 

34 Optional, 

35 TYPE_CHECKING, 

36 TypeVar, 

37 Union, 

38) 

39 

40from ..config import Config 

41from ..named import NamedValueSet 

42from ..utils import immutable 

43from .elements import Dimension, DimensionElement, SkyPixDimension 

44from .graph import DimensionGraph 

45from .config import processElementsConfig, processSkyPixConfig, DimensionConfig 

46from .packer import DimensionPackerFactory 

47 

48if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true

49 from .coordinate import ExpandedDataCoordinate 

50 from .packer import DimensionPacker 

51 

52 

53E = TypeVar("E", bound=DimensionElement) 

54 

55 

56@immutable 

57class DimensionUniverse(DimensionGraph): 

58 """A special `DimensionGraph` that constructs and manages a complete set of 

59 compatible dimensions. 

60 

61 `DimensionUniverse` is not a class-level singleton, but all instances are 

62 tracked in a singleton map keyed by the version number in the configuration 

63 they were loaded from. Because these universes are solely responsible for 

64 constructing `DimensionElement` instances, these are also indirectly 

65 tracked by that singleton as well. 

66 

67 Parameters 

68 ---------- 

69 config : `Config`, optional 

70 Configuration describing the dimensions and their relationships. If 

71 not provided, default configuration (from 

72 ``daf_butler/config/dimensions.yaml``) wil be loaded. 

73 """ 

74 

75 _instances: ClassVar[Dict[int, DimensionUniverse]] = {} 

76 """Singleton dictionary of all instances, keyed by version. 

77 

78 For internal use only. 

79 """ 

80 

81 def __new__(cls, config: Optional[Config] = None) -> DimensionUniverse: 

82 # Normalize the config and apply defaults. 

83 config = DimensionConfig(config) 

84 

85 # First see if an equivalent instance already exists. 

86 version = config["version"] 

87 self: Optional[DimensionUniverse] = cls._instances.get(version) 

88 if self is not None: 

89 return self 

90 

91 # Create the universe instance and add core attributes. 

92 # We don't want any of what DimensionGraph.__new__ does, so we just go 

93 # straight to object.__new__. The C++ side of my brain is offended by 

94 # this, but I think it's the right approach in Python, where we don't 

95 # have the option of having multiple constructors with different roles. 

96 self = object.__new__(cls) 

97 self.universe = self 

98 self._cache = {} 

99 self.dimensions = NamedValueSet() 

100 self.elements = NamedValueSet() 

101 

102 # Read the skypix dimensions from config. 

103 skyPixDimensions, self.commonSkyPix = processSkyPixConfig(config["skypix"]) 

104 # Add the skypix dimensions to the universe after sorting 

105 # lexicographically (no topological sort because skypix dimensions 

106 # never have any dependencies). 

107 for name in sorted(skyPixDimensions): 

108 skyPixDimensions[name]._finish(self, {}) 

109 

110 # Read the other dimension elements from config. 

111 elementsToDo = processElementsConfig(config["elements"]) 

112 # Add elements to the universe in topological order by identifying at 

113 # each outer iteration which elements have already had all of their 

114 # dependencies added. 

115 while elementsToDo: 

116 unblocked = [name for name, element in elementsToDo.items() 

117 if element._related.dependencies.isdisjoint(elementsToDo.keys())] 

118 unblocked.sort() # Break ties lexicographically. 

119 if not unblocked: 

120 raise RuntimeError(f"Cycle detected in dimension elements: {elementsToDo.keys()}.") 

121 for name in unblocked: 

122 # Finish initialization of the element with steps that 

123 # depend on those steps already having been run for all 

124 # dependencies. 

125 # This includes adding the element to self.elements and 

126 # (if appropriate) self.dimensions. 

127 elementsToDo.pop(name)._finish(self, elementsToDo) 

128 

129 # Add attributes for special subsets of the graph. 

130 self.empty = DimensionGraph(self, (), conform=False) 

131 self._finish() 

132 

133 # Set up factories for dataId packers as defined by config. 

134 # MyPy is totally confused by us setting attributes on self in __new__. 

135 self._packers = {} 

136 for name, subconfig in config.get("packers", {}).items(): 

137 self._packers[name] = DimensionPackerFactory.fromConfig(universe=self, # type: ignore 

138 config=subconfig) 

139 

140 # Use the version number from the config as a key in the singleton 

141 # dict containing all instances; that will let us transfer dimension 

142 # objects between processes using pickle without actually going 

143 # through real initialization, as long as a universe with the same 

144 # version has already been constructed in the receiving process. 

145 self._version = version 

146 cls._instances[self._version] = self 

147 return self 

148 

149 def __repr__(self) -> str: 

150 return f"DimensionUniverse({self})" 

151 

152 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

153 """Construct a `DimensionGraph` from a possibly-heterogenous iterable 

154 of `Dimension` instances and string names thereof. 

155 

156 Constructing `DimensionGraph` directly from names or dimension 

157 instances is slightly more efficient when it is known in advance that 

158 the iterable is not heterogenous. 

159 

160 Parameters 

161 ---------- 

162 iterable: iterable of `Dimension` or `str` 

163 Dimensions that must be included in the returned graph (their 

164 dependencies will be as well). 

165 

166 Returns 

167 ------- 

168 graph : `DimensionGraph` 

169 A `DimensionGraph` instance containing all given dimensions. 

170 """ 

171 names = set() 

172 for item in iterable: 

173 try: 

174 names.add(item.name) # type: ignore 

175 except AttributeError: 

176 names.add(item) 

177 return DimensionGraph(universe=self, names=names) 

178 

179 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

180 """Return a sorted version of the given iterable of dimension elements. 

181 

182 The universe's sort order is topological (an element's dependencies 

183 precede it), starting with skypix dimensions (which never have 

184 dependencies) and then sorting lexicographically to break ties. 

185 

186 Parameters 

187 ---------- 

188 elements : iterable of `DimensionElement`. 

189 Elements to be sorted. 

190 reverse : `bool`, optional 

191 If `True`, sort in the opposite order. 

192 

193 Returns 

194 ------- 

195 sorted : `list` of `DimensionElement` 

196 A sorted list containing the same elements that were given. 

197 """ 

198 s = set(elements) 

199 result = [element for element in self.elements if element in s or element.name in s] 

200 if reverse: 

201 result.reverse() 

202 # mypy thinks this can return DimensionElements even if all the user 

203 # passed it was Dimensions; we know better. 

204 return result # type: ignore 

205 

206 def makePacker(self, name: str, dataId: ExpandedDataCoordinate) -> DimensionPacker: 

207 """Construct a `DimensionPacker` that can pack data ID dictionaries 

208 into unique integers. 

209 

210 Parameters 

211 ---------- 

212 name : `str` 

213 Name of the packer, matching a key in the "packers" section of the 

214 dimension configuration. 

215 dataId : `ExpandedDataCoordinate` 

216 Fully-expanded data ID that identfies the at least the "fixed" 

217 dimensions of the packer (i.e. those that are assumed/given, 

218 setting the space over which packed integer IDs are unique). 

219 """ 

220 return self._packers[name](dataId) 

221 

222 def getEncodeLength(self) -> int: 

223 """Return the size (in bytes) of the encoded size of `DimensionGraph` 

224 instances in this universe. 

225 

226 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

227 information. 

228 """ 

229 return math.ceil(len(self.dimensions)/8) 

230 

231 @classmethod 

232 def _unpickle(cls, version: int) -> DimensionUniverse: 

233 """Callable used for unpickling. 

234 

235 For internal use only. 

236 """ 

237 try: 

238 return cls._instances[version] 

239 except KeyError as err: 

240 raise pickle.UnpicklingError( 

241 f"DimensionUniverse with version '{version}' " 

242 f"not found. Note that DimensionUniverse objects are not " 

243 f"truly serialized; when using pickle to transfer them " 

244 f"between processes, an equivalent instance with the same " 

245 f"version must already exist in the receiving process." 

246 ) from err 

247 

248 def __reduce__(self) -> tuple: 

249 return (self._unpickle, (self._version,)) 

250 

251 # Class attributes below are shadowed by instance attributes, and are 

252 # present just to hold the docstrings for those instance attributes. 

253 

254 empty: DimensionGraph 

255 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

256 """ 

257 

258 commonSkyPix: SkyPixDimension 

259 """The special skypix dimension that is used to relate all other spatial 

260 dimensions in the `Registry` database (`SkyPixDimension`). 

261 """ 

262 

263 _cache: Dict[FrozenSet[str], DimensionGraph] 

264 

265 _packers: Dict[str, DimensionPackerFactory] 

266 

267 _version: int