Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 ClassVar, 

30 Dict, 

31 FrozenSet, 

32 Iterable, 

33 List, 

34 Mapping, 

35 Optional, 

36 Set, 

37 TYPE_CHECKING, 

38 TypeVar, 

39 Union, 

40) 

41 

42from ..config import Config 

43from ..named import NamedValueAbstractSet, NamedValueSet 

44from .._topology import TopologicalSpace, TopologicalFamily 

45from ..utils import cached_getter, immutable 

46from ._config import DimensionConfig 

47from ._elements import Dimension, DimensionElement 

48from ._graph import DimensionGraph 

49from ._governor import GovernorDimension 

50from ._database import DatabaseDimensionElement 

51from ._skypix import SkyPixDimension, SkyPixSystem 

52 

53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true

54 from ._coordinate import DataCoordinate 

55 from ._packer import DimensionPacker, DimensionPackerFactory 

56 from .construction import DimensionConstructionBuilder 

57 

58 

59E = TypeVar("E", bound=DimensionElement) 

60 

61 

62@immutable 

63class DimensionUniverse: 

64 """A parent class that represents a complete, self-consistent set of 

65 dimensions and their relationships. 

66 

67 `DimensionUniverse` is not a class-level singleton, but all instances are 

68 tracked in a singleton map keyed by the version number in the configuration 

69 they were loaded from. Because these universes are solely responsible for 

70 constructing `DimensionElement` instances, these are also indirectly 

71 tracked by that singleton as well. 

72 

73 Parameters 

74 ---------- 

75 config : `Config`, optional 

76 Configuration object from which dimension definitions can be extracted. 

77 Ignored if ``builder`` is provided, or if ``version`` is provided and 

78 an instance with that version already exists. 

79 version : `int`, optional 

80 Integer version for this `DimensionUniverse`. If not provided, a 

81 version will be obtained from ``builder`` or ``config``. 

82 builder : `DimensionConstructionBuilder`, optional 

83 Builder object used to initialize a new instance. Ignored if 

84 ``version`` is provided and an instance with that version already 

85 exists. Should not have had `~DimensionConstructionBuilder.finish` 

86 called; this will be called if needed by `DimensionUniverse`. 

87 """ 

88 

89 _instances: ClassVar[Dict[int, DimensionUniverse]] = {} 

90 """Singleton dictionary of all instances, keyed by version. 

91 

92 For internal use only. 

93 """ 

94 

95 def __new__( 

96 cls, 

97 config: Optional[Config] = None, *, 

98 version: Optional[int] = None, 

99 builder: Optional[DimensionConstructionBuilder] = None, 

100 ) -> DimensionUniverse: 

101 # Try to get a version first, to look for existing instances; try to 

102 # do as little work as possible at this stage. 

103 if version is None: 

104 if builder is None: 

105 config = DimensionConfig(config) 

106 version = config["version"] 

107 else: 

108 version = builder.version 

109 

110 # See if an equivalent instance already exists. 

111 self: Optional[DimensionUniverse] = cls._instances.get(version) 

112 if self is not None: 

113 return self 

114 

115 # Ensure we have a builder, building one from config if necessary. 

116 if builder is None: 

117 config = DimensionConfig(config) 

118 builder = config.makeBuilder() 

119 

120 # Delegate to the builder for most of the construction work. 

121 builder.finish() 

122 

123 # Create the universe instance and create core attributes, mostly 

124 # copying from builder. 

125 self = object.__new__(cls) 

126 assert self is not None 

127 self._cache = {} 

128 self._dimensions = builder.dimensions 

129 self._elements = builder.elements 

130 self._topology = builder.topology 

131 self._packers = builder.packers 

132 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

133 assert isinstance(commonSkyPix, SkyPixDimension) 

134 self.commonSkyPix = commonSkyPix 

135 

136 # Attach self to all elements. 

137 for element in self._elements: 

138 element.universe = self 

139 

140 # Add attribute for special subsets of the graph. 

141 self.empty = DimensionGraph(self, (), conform=False) 

142 

143 # Use the version number from the config as a key in the singleton 

144 # dict containing all instances; that will let us transfer dimension 

145 # objects between processes using pickle without actually going 

146 # through real initialization, as long as a universe with the same 

147 # version has already been constructed in the receiving process. 

148 self._version = version 

149 cls._instances[self._version] = self 

150 

151 # Build mappings from element to index. These are used for 

152 # topological-sort comparison operators in DimensionElement itself. 

153 self._elementIndices = { 

154 name: i for i, name in enumerate(self._elements.names) 

155 } 

156 # Same for dimension to index, sorted topologically across required 

157 # and implied. This is used for encode/decode. 

158 self._dimensionIndices = { 

159 name: i for i, name in enumerate(self._dimensions.names) 

160 } 

161 

162 return self 

163 

164 def __repr__(self) -> str: 

165 return f"DimensionUniverse({self._version})" 

166 

167 def __getitem__(self, name: str) -> DimensionElement: 

168 return self._elements[name] 

169 

170 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

171 """Return the `DimensionElement` with the given name or a default. 

172 

173 Parameters 

174 ---------- 

175 name : `str` 

176 Name of the element. 

177 default : `DimensionElement`, optional 

178 Element to return if the named one does not exist. Defaults to 

179 `None`. 

180 

181 Returns 

182 ------- 

183 element : `DimensionElement` 

184 The named element. 

185 """ 

186 return self._elements.get(name, default) 

187 

188 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

189 """Return a set of all static elements in this universe. 

190 

191 Non-static elements that are created as needed may also exist, but 

192 these are guaranteed to have no direct relationships to other elements 

193 (though they may have spatial or temporal relationships). 

194 

195 Returns 

196 ------- 

197 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

198 A frozen set of `DimensionElement` instances. 

199 """ 

200 return self._elements 

201 

202 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

203 """Return a set of all static dimensions in this universe. 

204 

205 Non-static dimensions that are created as needed may also exist, but 

206 these are guaranteed to have no direct relationships to other elements 

207 (though they may have spatial or temporal relationships). 

208 

209 Returns 

210 ------- 

211 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

212 A frozen set of `Dimension` instances. 

213 """ 

214 return self._dimensions 

215 

216 @cached_getter 

217 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

218 """Return a set of all `GovernorDimension` instances in this universe. 

219 

220 Returns 

221 ------- 

222 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

223 A frozen set of `GovernorDimension` instances. 

224 """ 

225 return NamedValueSet( 

226 d for d in self._dimensions if isinstance(d, GovernorDimension) 

227 ).freeze() 

228 

229 @cached_getter 

230 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

231 """Return a set of all `DatabaseDimensionElement` instances in this 

232 universe. 

233 

234 This does not include `GovernorDimension` instances, which are backed 

235 by the database but do not inherit from `DatabaseDimensionElement`. 

236 

237 Returns 

238 ------- 

239 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

240 A frozen set of `DatabaseDimensionElement` instances. 

241 """ 

242 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

243 

244 @property # type: ignore 

245 @cached_getter 

246 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

247 """All skypix systems known to this universe 

248 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

249 """ 

250 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL] 

251 if isinstance(family, SkyPixSystem)]).freeze() 

252 

253 def getElementIndex(self, name: str) -> int: 

254 """Return the position of the named dimension element in this 

255 universe's sorting of all elements. 

256 

257 Parameters 

258 ---------- 

259 name : `str` 

260 Name of the element. 

261 

262 Returns 

263 ------- 

264 index : `int` 

265 Sorting index for this element. 

266 """ 

267 return self._elementIndices[name] 

268 

269 def getDimensionIndex(self, name: str) -> int: 

270 """Return the position of the named dimension in this universe's 

271 sorting of all dimensions. 

272 

273 Parameters 

274 ---------- 

275 name : `str` 

276 Name of the dimension. 

277 

278 Returns 

279 ------- 

280 index : `int` 

281 Sorting index for this dimension. 

282 

283 Notes 

284 ----- 

285 The dimension sort order for a universe is consistent with the element 

286 order (all dimensions are elements), and either can be used to sort 

287 dimensions if used consistently. But there are also some contexts in 

288 which contiguous dimension-only indices are necessary or at least 

289 desirable. 

290 """ 

291 return self._dimensionIndices[name] 

292 

293 def expandDimensionNameSet(self, names: Set[str]) -> None: 

294 """Expand a set of dimension names in-place to include recursive 

295 dependencies. 

296 

297 This is an advanced interface for cases where constructing a 

298 `DimensionGraph` (which also expands required dependencies) is 

299 impossible or undesirable. 

300 

301 Parameters 

302 ---------- 

303 names : `set` [ `str` ] 

304 A true `set` of dimension names, to be expanded in-place. 

305 """ 

306 # Keep iterating until the set of names stops growing. This is not as 

307 # efficient as it could be, but we work pretty hard cache 

308 # DimensionGraph instances to keep actual construction rare, so that 

309 # shouldn't matter. 

310 oldSize = len(names) 

311 while True: 

312 # iterate over a temporary copy so we can modify the original 

313 for name in tuple(names): 

314 names.update(self._dimensions[name].required.names) 

315 names.update(self._dimensions[name].implied.names) 

316 if oldSize == len(names): 

317 break 

318 else: 

319 oldSize = len(names) 

320 

321 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

322 """Construct a `DimensionGraph` from a possibly-heterogenous iterable 

323 of `Dimension` instances and string names thereof. 

324 

325 Constructing `DimensionGraph` directly from names or dimension 

326 instances is slightly more efficient when it is known in advance that 

327 the iterable is not heterogenous. 

328 

329 Parameters 

330 ---------- 

331 iterable: iterable of `Dimension` or `str` 

332 Dimensions that must be included in the returned graph (their 

333 dependencies will be as well). 

334 

335 Returns 

336 ------- 

337 graph : `DimensionGraph` 

338 A `DimensionGraph` instance containing all given dimensions. 

339 """ 

340 names = set() 

341 for item in iterable: 

342 try: 

343 names.add(item.name) # type: ignore 

344 except AttributeError: 

345 names.add(item) 

346 return DimensionGraph(universe=self, names=names) 

347 

348 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

349 """Return a sorted version of the given iterable of dimension elements. 

350 

351 The universe's sort order is topological (an element's dependencies 

352 precede it), with an unspecified (but deterministic) approach to 

353 breaking ties. 

354 

355 Parameters 

356 ---------- 

357 elements : iterable of `DimensionElement`. 

358 Elements to be sorted. 

359 reverse : `bool`, optional 

360 If `True`, sort in the opposite order. 

361 

362 Returns 

363 ------- 

364 sorted : `list` of `DimensionElement` 

365 A sorted list containing the same elements that were given. 

366 """ 

367 s = set(elements) 

368 result = [element for element in self._elements if element in s or element.name in s] 

369 if reverse: 

370 result.reverse() 

371 # mypy thinks this can return DimensionElements even if all the user 

372 # passed it was Dimensions; we know better. 

373 return result # type: ignore 

374 

375 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

376 """Construct a `DimensionPacker` that can pack data ID dictionaries 

377 into unique integers. 

378 

379 Parameters 

380 ---------- 

381 name : `str` 

382 Name of the packer, matching a key in the "packers" section of the 

383 dimension configuration. 

384 dataId : `DataCoordinate` 

385 Fully-expanded data ID that identfies the at least the "fixed" 

386 dimensions of the packer (i.e. those that are assumed/given, 

387 setting the space over which packed integer IDs are unique). 

388 ``dataId.hasRecords()`` must return `True`. 

389 """ 

390 return self._packers[name](self, dataId) 

391 

392 def getEncodeLength(self) -> int: 

393 """Return the size (in bytes) of the encoded size of `DimensionGraph` 

394 instances in this universe. 

395 

396 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

397 information. 

398 """ 

399 return math.ceil(len(self._dimensions)/8) 

400 

401 @classmethod 

402 def _unpickle(cls, version: int) -> DimensionUniverse: 

403 """Callable used for unpickling. 

404 

405 For internal use only. 

406 """ 

407 try: 

408 return cls._instances[version] 

409 except KeyError as err: 

410 raise pickle.UnpicklingError( 

411 f"DimensionUniverse with version '{version}' " 

412 f"not found. Note that DimensionUniverse objects are not " 

413 f"truly serialized; when using pickle to transfer them " 

414 f"between processes, an equivalent instance with the same " 

415 f"version must already exist in the receiving process." 

416 ) from err 

417 

418 def __reduce__(self) -> tuple: 

419 return (self._unpickle, (self._version,)) 

420 

421 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

422 # DimensionUniverse is recursively immutable; see note in @immutable 

423 # decorator. 

424 return self 

425 

426 # Class attributes below are shadowed by instance attributes, and are 

427 # present just to hold the docstrings for those instance attributes. 

428 

429 empty: DimensionGraph 

430 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

431 """ 

432 

433 commonSkyPix: SkyPixDimension 

434 """The special skypix dimension that is used to relate all other spatial 

435 dimensions in the `Registry` database (`SkyPixDimension`). 

436 """ 

437 

438 _cache: Dict[FrozenSet[str], DimensionGraph] 

439 

440 _dimensions: NamedValueAbstractSet[Dimension] 

441 

442 _elements: NamedValueAbstractSet[DimensionElement] 

443 

444 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

445 

446 _dimensionIndices: Dict[str, int] 

447 

448 _elementIndices: Dict[str, int] 

449 

450 _packers: Dict[str, DimensionPackerFactory] 

451 

452 _version: int