Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 ClassVar, 

30 Dict, 

31 FrozenSet, 

32 Iterable, 

33 List, 

34 Optional, 

35 TYPE_CHECKING, 

36 TypeVar, 

37 Union, 

38) 

39 

40from ..config import Config 

41from ..named import NamedValueSet 

42from ..utils import immutable 

43from .elements import Dimension, DimensionElement, SkyPixDimension 

44from .graph import DimensionGraph 

45from .config import processElementsConfig, processSkyPixConfig, DimensionConfig 

46from .packer import DimensionPackerFactory 

47 

48if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true

49 from .coordinate import DataCoordinate 

50 from .packer import DimensionPacker 

51 

52 

53E = TypeVar("E", bound=DimensionElement) 

54 

55 

56@immutable 

57class DimensionUniverse: 

58 """A parent class that represents a complete, self-consistent set of 

59 dimensions and their relationships. 

60 

61 `DimensionUniverse` is not a class-level singleton, but all instances are 

62 tracked in a singleton map keyed by the version number in the configuration 

63 they were loaded from. Because these universes are solely responsible for 

64 constructing `DimensionElement` instances, these are also indirectly 

65 tracked by that singleton as well. 

66 

67 Parameters 

68 ---------- 

69 config : `Config`, optional 

70 Configuration describing the dimensions and their relationships. If 

71 not provided, default configuration (from 

72 ``daf_butler/config/dimensions.yaml``) wil be loaded. 

73 """ 

74 

75 _instances: ClassVar[Dict[int, DimensionUniverse]] = {} 

76 """Singleton dictionary of all instances, keyed by version. 

77 

78 For internal use only. 

79 """ 

80 

81 def __new__(cls, config: Optional[Config] = None) -> DimensionUniverse: 

82 # Normalize the config and apply defaults. 

83 config = DimensionConfig(config) 

84 

85 # First see if an equivalent instance already exists. 

86 version = config["version"] 

87 self: Optional[DimensionUniverse] = cls._instances.get(version) 

88 if self is not None: 

89 return self 

90 

91 # Create the universe instance and add core attributes. 

92 self = object.__new__(cls) 

93 assert self is not None 

94 self._cache = {} 

95 self._dimensions = NamedValueSet() 

96 self._elements = NamedValueSet() 

97 

98 # Read the skypix dimensions from config. 

99 skyPixDimensions, self.commonSkyPix = processSkyPixConfig(config["skypix"]) 

100 # Add the skypix dimensions to the universe after sorting 

101 # lexicographically (no topological sort because skypix dimensions 

102 # never have any dependencies). 

103 for name in sorted(skyPixDimensions): 

104 skyPixDimensions[name]._finish(self, {}) 

105 

106 # Read the other dimension elements from config. 

107 elementsToDo = processElementsConfig(config["elements"]) 

108 # Add elements to the universe in topological order by identifying at 

109 # each outer iteration which elements have already had all of their 

110 # dependencies added. 

111 while elementsToDo: 

112 unblocked = [name for name, element in elementsToDo.items() 

113 if element._related.dependencies.isdisjoint(elementsToDo.keys())] 

114 unblocked.sort() # Break ties lexicographically. 

115 if not unblocked: 

116 raise RuntimeError(f"Cycle detected in dimension elements: {elementsToDo.keys()}.") 

117 for name in unblocked: 

118 # Finish initialization of the element with steps that 

119 # depend on those steps already having been run for all 

120 # dependencies. 

121 # This includes adding the element to self.elements and 

122 # (if appropriate) self.dimensions. 

123 elementsToDo.pop(name)._finish(self, elementsToDo) 

124 

125 # Add attributes for special subsets of the graph. 

126 self.empty = DimensionGraph(self, (), conform=False) 

127 

128 # Set up factories for dataId packers as defined by config. 

129 # MyPy is totally confused by us setting attributes on self in __new__. 

130 self._packers = {} 

131 for name, subconfig in config.get("packers", {}).items(): 

132 self._packers[name] = DimensionPackerFactory.fromConfig(universe=self, # type: ignore 

133 config=subconfig) 

134 

135 # Use the version number from the config as a key in the singleton 

136 # dict containing all instances; that will let us transfer dimension 

137 # objects between processes using pickle without actually going 

138 # through real initialization, as long as a universe with the same 

139 # version has already been constructed in the receiving process. 

140 self._version = version 

141 cls._instances[self._version] = self 

142 

143 # Build mappings from element to index. These are used for 

144 # topological-sort comparison operators in DimensionElement itself. 

145 self._elementIndices = { 

146 name: i for i, name in enumerate(self._elements.names) 

147 } 

148 # Same for dimension to index, sorted topologically across required 

149 # and implied. This is used for encode/decode. 

150 self._dimensionIndices = { 

151 name: i for i, name in enumerate(self._dimensions.names) 

152 } 

153 

154 # Freeze internal sets so we can return them in methods without 

155 # worrying about modifications. 

156 self._elements.freeze() 

157 self._dimensions.freeze() 

158 return self 

159 

160 def __repr__(self) -> str: 

161 return f"DimensionUniverse({self})" 

162 

163 def __getitem__(self, name: str) -> DimensionElement: 

164 return self._elements[name] 

165 

166 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

167 """Return the `DimensionElement` with the given name or a default. 

168 

169 Parameters 

170 ---------- 

171 name : `str` 

172 Name of the element. 

173 default : `DimensionElement`, optional 

174 Element to return if the named one does not exist. Defaults to 

175 `None`. 

176 

177 Returns 

178 ------- 

179 element : `DimensionElement` 

180 The named element. 

181 """ 

182 return self._elements.get(name, default) 

183 

184 def getStaticElements(self) -> NamedValueSet[DimensionElement]: 

185 """Return a set of all static elements in this universe. 

186 

187 Non-static elements that are created as needed may also exist, but 

188 these are guaranteed to have no direct relationships to other elements 

189 (though they may have spatial or temporal relationships). 

190 

191 Returns 

192 ------- 

193 elements : `NamedValueSet` [ `DimensionElement` ] 

194 A frozen set of `DimensionElement` instances. 

195 """ 

196 return self._elements 

197 

198 def getStaticDimensions(self) -> NamedValueSet[Dimension]: 

199 """Return a set of all static dimensions in this universe. 

200 

201 Non-static dimensions that are created as needed may also exist, but 

202 these are guaranteed to have no direct relationships to other elements 

203 (though they may have spatial or temporal relationships). 

204 

205 Returns 

206 ------- 

207 dimensions : `NamedValueSet` [ `Dimension` ] 

208 A frozen set of `Dimension` instances. 

209 """ 

210 return self._dimensions 

211 

212 def getElementIndex(self, name: str) -> int: 

213 """Return the position of the named dimension element in this 

214 universe's sorting of all elements. 

215 

216 Parameters 

217 ---------- 

218 name : `str` 

219 Name of the element. 

220 

221 Returns 

222 ------- 

223 index : `int` 

224 Sorting index for this element. 

225 """ 

226 return self._elementIndices[name] 

227 

228 def getDimensionIndex(self, name: str) -> int: 

229 """Return the position of the named dimension in this universe's 

230 sorting of all dimensions. 

231 

232 Parameters 

233 ---------- 

234 name : `str` 

235 Name of the dimension. 

236 

237 Returns 

238 ------- 

239 index : `int` 

240 Sorting index for this dimension. 

241 

242 Notes 

243 ----- 

244 The dimension sort order for a universe is consistent with the element 

245 order (all dimensions are elements), and either can be used to sort 

246 dimensions if used consistently. But there are also some contexts in 

247 which contiguous dimension-only indices are necessary or at least 

248 desirable. 

249 """ 

250 return self._dimensionIndices[name] 

251 

252 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

253 """Construct a `DimensionGraph` from a possibly-heterogenous iterable 

254 of `Dimension` instances and string names thereof. 

255 

256 Constructing `DimensionGraph` directly from names or dimension 

257 instances is slightly more efficient when it is known in advance that 

258 the iterable is not heterogenous. 

259 

260 Parameters 

261 ---------- 

262 iterable: iterable of `Dimension` or `str` 

263 Dimensions that must be included in the returned graph (their 

264 dependencies will be as well). 

265 

266 Returns 

267 ------- 

268 graph : `DimensionGraph` 

269 A `DimensionGraph` instance containing all given dimensions. 

270 """ 

271 names = set() 

272 for item in iterable: 

273 try: 

274 names.add(item.name) # type: ignore 

275 except AttributeError: 

276 names.add(item) 

277 return DimensionGraph(universe=self, names=names) 

278 

279 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

280 """Return a sorted version of the given iterable of dimension elements. 

281 

282 The universe's sort order is topological (an element's dependencies 

283 precede it), with an unspecified (but deterministic) approach to 

284 breaking ties. 

285 

286 Parameters 

287 ---------- 

288 elements : iterable of `DimensionElement`. 

289 Elements to be sorted. 

290 reverse : `bool`, optional 

291 If `True`, sort in the opposite order. 

292 

293 Returns 

294 ------- 

295 sorted : `list` of `DimensionElement` 

296 A sorted list containing the same elements that were given. 

297 """ 

298 s = set(elements) 

299 result = [element for element in self._elements if element in s or element.name in s] 

300 if reverse: 

301 result.reverse() 

302 # mypy thinks this can return DimensionElements even if all the user 

303 # passed it was Dimensions; we know better. 

304 return result # type: ignore 

305 

306 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

307 """Construct a `DimensionPacker` that can pack data ID dictionaries 

308 into unique integers. 

309 

310 Parameters 

311 ---------- 

312 name : `str` 

313 Name of the packer, matching a key in the "packers" section of the 

314 dimension configuration. 

315 dataId : `DataCoordinate` 

316 Fully-expanded data ID that identfies the at least the "fixed" 

317 dimensions of the packer (i.e. those that are assumed/given, 

318 setting the space over which packed integer IDs are unique). 

319 ``dataId.hasRecords()`` must return `True`. 

320 """ 

321 return self._packers[name](dataId) 

322 

323 def getEncodeLength(self) -> int: 

324 """Return the size (in bytes) of the encoded size of `DimensionGraph` 

325 instances in this universe. 

326 

327 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

328 information. 

329 """ 

330 return math.ceil(len(self._dimensions)/8) 

331 

332 @classmethod 

333 def _unpickle(cls, version: int) -> DimensionUniverse: 

334 """Callable used for unpickling. 

335 

336 For internal use only. 

337 """ 

338 try: 

339 return cls._instances[version] 

340 except KeyError as err: 

341 raise pickle.UnpicklingError( 

342 f"DimensionUniverse with version '{version}' " 

343 f"not found. Note that DimensionUniverse objects are not " 

344 f"truly serialized; when using pickle to transfer them " 

345 f"between processes, an equivalent instance with the same " 

346 f"version must already exist in the receiving process." 

347 ) from err 

348 

349 def __reduce__(self) -> tuple: 

350 return (self._unpickle, (self._version,)) 

351 

352 # Class attributes below are shadowed by instance attributes, and are 

353 # present just to hold the docstrings for those instance attributes. 

354 

355 empty: DimensionGraph 

356 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

357 """ 

358 

359 commonSkyPix: SkyPixDimension 

360 """The special skypix dimension that is used to relate all other spatial 

361 dimensions in the `Registry` database (`SkyPixDimension`). 

362 """ 

363 

364 _cache: Dict[FrozenSet[str], DimensionGraph] 

365 

366 _dimensions: NamedValueSet[Dimension] 

367 

368 _elements: NamedValueSet[DimensionElement] 

369 

370 _dimensionIndices: Dict[str, int] 

371 

372 _elementIndices: Dict[str, int] 

373 

374 _packers: Dict[str, DimensionPackerFactory] 

375 

376 _version: int