Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 ClassVar, 

30 Dict, 

31 FrozenSet, 

32 Iterable, 

33 List, 

34 Mapping, 

35 Optional, 

36 Set, 

37 TYPE_CHECKING, 

38 TypeVar, 

39 Union, 

40) 

41 

42from ..config import Config 

43from ..named import NamedValueAbstractSet, NamedValueSet 

44from .._topology import TopologicalSpace, TopologicalFamily 

45from ..utils import cached_getter, immutable 

46from ._config import DimensionConfig 

47from ._elements import Dimension, DimensionElement 

48from ._graph import DimensionGraph 

49from ._governor import GovernorDimension 

50from ._database import DatabaseDimensionElement 

51from ._skypix import SkyPixDimension, SkyPixSystem 

52 

53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true

54 from ._coordinate import DataCoordinate 

55 from ._packer import DimensionPacker, DimensionPackerFactory 

56 from .construction import DimensionConstructionBuilder 

57 

58 

59E = TypeVar("E", bound=DimensionElement) 

60 

61 

62@immutable 

63class DimensionUniverse: 

64 """A parent class that represents a complete, self-consistent set of 

65 dimensions and their relationships. 

66 

67 `DimensionUniverse` is not a class-level singleton, but all instances are 

68 tracked in a singleton map keyed by the version number in the configuration 

69 they were loaded from. Because these universes are solely responsible for 

70 constructing `DimensionElement` instances, these are also indirectly 

71 tracked by that singleton as well. 

72 

73 Parameters 

74 ---------- 

75 config : `Config`, optional 

76 Configuration object from which dimension definitions can be extracted. 

77 Ignored if ``builder`` is provided, or if ``version`` is provided and 

78 an instance with that version already exists. 

79 version : `int`, optional 

80 Integer version for this `DimensionUniverse`. If not provided, a 

81 version will be obtained from ``builder`` or ``config``. 

82 builder : `DimensionConstructionBuilder`, optional 

83 Builder object used to initialize a new instance. Ignored if 

84 ``version`` is provided and an instance with that version already 

85 exists. Should not have had `~DimensionConstructionBuilder.finish` 

86 called; this will be called if needed by `DimensionUniverse`. 

87 """ 

88 

89 _instances: ClassVar[Dict[int, DimensionUniverse]] = {} 

90 """Singleton dictionary of all instances, keyed by version. 

91 

92 For internal use only. 

93 """ 

94 

95 def __new__( 

96 cls, 

97 config: Optional[Config] = None, *, 

98 version: Optional[int] = None, 

99 builder: Optional[DimensionConstructionBuilder] = None, 

100 ) -> DimensionUniverse: 

101 # Try to get a version first, to look for existing instances; try to 

102 # do as little work as possible at this stage. 

103 if version is None: 

104 if builder is None: 

105 config = DimensionConfig(config) 

106 version = config["version"] 

107 else: 

108 version = builder.version 

109 

110 # See if an equivalent instance already exists. 

111 self: Optional[DimensionUniverse] = cls._instances.get(version) 

112 if self is not None: 

113 return self 

114 

115 # Ensure we have a builder, building one from config if necessary. 

116 if builder is None: 

117 config = DimensionConfig(config) 

118 builder = config.makeBuilder() 

119 

120 # Delegate to the builder for most of the construction work. 

121 builder.finish() 

122 

123 # Create the universe instance and create core attributes, mostly 

124 # copying from builder. 

125 self = object.__new__(cls) 

126 assert self is not None 

127 self._cache = {} 

128 self._dimensions = builder.dimensions 

129 self._elements = builder.elements 

130 self._topology = builder.topology 

131 self._packers = builder.packers 

132 self.commonSkyPix = self._dimensions[builder.commonSkyPixName] 

133 

134 # Attach self to all elements. 

135 for element in self._elements: 

136 element.universe = self 

137 

138 # Add attribute for special subsets of the graph. 

139 self.empty = DimensionGraph(self, (), conform=False) 

140 

141 # Use the version number from the config as a key in the singleton 

142 # dict containing all instances; that will let us transfer dimension 

143 # objects between processes using pickle without actually going 

144 # through real initialization, as long as a universe with the same 

145 # version has already been constructed in the receiving process. 

146 self._version = version 

147 cls._instances[self._version] = self 

148 

149 # Build mappings from element to index. These are used for 

150 # topological-sort comparison operators in DimensionElement itself. 

151 self._elementIndices = { 

152 name: i for i, name in enumerate(self._elements.names) 

153 } 

154 # Same for dimension to index, sorted topologically across required 

155 # and implied. This is used for encode/decode. 

156 self._dimensionIndices = { 

157 name: i for i, name in enumerate(self._dimensions.names) 

158 } 

159 

160 return self 

161 

162 def __repr__(self) -> str: 

163 return f"DimensionUniverse({self._version})" 

164 

165 def __getitem__(self, name: str) -> DimensionElement: 

166 return self._elements[name] 

167 

168 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

169 """Return the `DimensionElement` with the given name or a default. 

170 

171 Parameters 

172 ---------- 

173 name : `str` 

174 Name of the element. 

175 default : `DimensionElement`, optional 

176 Element to return if the named one does not exist. Defaults to 

177 `None`. 

178 

179 Returns 

180 ------- 

181 element : `DimensionElement` 

182 The named element. 

183 """ 

184 return self._elements.get(name, default) 

185 

186 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

187 """Return a set of all static elements in this universe. 

188 

189 Non-static elements that are created as needed may also exist, but 

190 these are guaranteed to have no direct relationships to other elements 

191 (though they may have spatial or temporal relationships). 

192 

193 Returns 

194 ------- 

195 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

196 A frozen set of `DimensionElement` instances. 

197 """ 

198 return self._elements 

199 

200 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

201 """Return a set of all static dimensions in this universe. 

202 

203 Non-static dimensions that are created as needed may also exist, but 

204 these are guaranteed to have no direct relationships to other elements 

205 (though they may have spatial or temporal relationships). 

206 

207 Returns 

208 ------- 

209 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

210 A frozen set of `Dimension` instances. 

211 """ 

212 return self._dimensions 

213 

214 @cached_getter 

215 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

216 """Return a set of all `GovernorDimension` instances in this universe. 

217 

218 Returns 

219 ------- 

220 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

221 A frozen set of `GovernorDimension` instances. 

222 """ 

223 return NamedValueSet( 

224 d for d in self._dimensions if isinstance(d, GovernorDimension) 

225 ).freeze() 

226 

227 @cached_getter 

228 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

229 """Return a set of all `DatabaseDimensionElement` instances in this 

230 universe. 

231 

232 This does not include `GovernorDimension` instances, which are backed 

233 by the database but do not inherit from `DatabaseDimensionElement`. 

234 

235 Returns 

236 ------- 

237 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

238 A frozen set of `DatabaseDimensionElement` instances. 

239 """ 

240 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

241 

242 @property # type: ignore 

243 @cached_getter 

244 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

245 """All skypix systems known to this universe 

246 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

247 """ 

248 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL] 

249 if isinstance(family, SkyPixSystem)]).freeze() 

250 

251 def getElementIndex(self, name: str) -> int: 

252 """Return the position of the named dimension element in this 

253 universe's sorting of all elements. 

254 

255 Parameters 

256 ---------- 

257 name : `str` 

258 Name of the element. 

259 

260 Returns 

261 ------- 

262 index : `int` 

263 Sorting index for this element. 

264 """ 

265 return self._elementIndices[name] 

266 

267 def getDimensionIndex(self, name: str) -> int: 

268 """Return the position of the named dimension in this universe's 

269 sorting of all dimensions. 

270 

271 Parameters 

272 ---------- 

273 name : `str` 

274 Name of the dimension. 

275 

276 Returns 

277 ------- 

278 index : `int` 

279 Sorting index for this dimension. 

280 

281 Notes 

282 ----- 

283 The dimension sort order for a universe is consistent with the element 

284 order (all dimensions are elements), and either can be used to sort 

285 dimensions if used consistently. But there are also some contexts in 

286 which contiguous dimension-only indices are necessary or at least 

287 desirable. 

288 """ 

289 return self._dimensionIndices[name] 

290 

291 def expandDimensionNameSet(self, names: Set[str]) -> None: 

292 """Expand a set of dimension names in-place to include recursive 

293 dependencies. 

294 

295 This is an advanced interface for cases where constructing a 

296 `DimensionGraph` (which also expands required dependencies) is 

297 impossible or undesirable. 

298 

299 Parameters 

300 ---------- 

301 names : `set` [ `str` ] 

302 A true `set` of dimension names, to be expanded in-place. 

303 """ 

304 # Keep iterating until the set of names stops growing. This is not as 

305 # efficient as it could be, but we work pretty hard cache 

306 # DimensionGraph instances to keep actual construction rare, so that 

307 # shouldn't matter. 

308 oldSize = len(names) 

309 while True: 

310 # iterate over a temporary copy so we can modify the original 

311 for name in tuple(names): 

312 names.update(self._dimensions[name].required.names) 

313 names.update(self._dimensions[name].implied.names) 

314 if oldSize == len(names): 

315 break 

316 else: 

317 oldSize = len(names) 

318 

319 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

320 """Construct a `DimensionGraph` from a possibly-heterogenous iterable 

321 of `Dimension` instances and string names thereof. 

322 

323 Constructing `DimensionGraph` directly from names or dimension 

324 instances is slightly more efficient when it is known in advance that 

325 the iterable is not heterogenous. 

326 

327 Parameters 

328 ---------- 

329 iterable: iterable of `Dimension` or `str` 

330 Dimensions that must be included in the returned graph (their 

331 dependencies will be as well). 

332 

333 Returns 

334 ------- 

335 graph : `DimensionGraph` 

336 A `DimensionGraph` instance containing all given dimensions. 

337 """ 

338 names = set() 

339 for item in iterable: 

340 try: 

341 names.add(item.name) # type: ignore 

342 except AttributeError: 

343 names.add(item) 

344 return DimensionGraph(universe=self, names=names) 

345 

346 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

347 """Return a sorted version of the given iterable of dimension elements. 

348 

349 The universe's sort order is topological (an element's dependencies 

350 precede it), with an unspecified (but deterministic) approach to 

351 breaking ties. 

352 

353 Parameters 

354 ---------- 

355 elements : iterable of `DimensionElement`. 

356 Elements to be sorted. 

357 reverse : `bool`, optional 

358 If `True`, sort in the opposite order. 

359 

360 Returns 

361 ------- 

362 sorted : `list` of `DimensionElement` 

363 A sorted list containing the same elements that were given. 

364 """ 

365 s = set(elements) 

366 result = [element for element in self._elements if element in s or element.name in s] 

367 if reverse: 

368 result.reverse() 

369 # mypy thinks this can return DimensionElements even if all the user 

370 # passed it was Dimensions; we know better. 

371 return result # type: ignore 

372 

373 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

374 """Construct a `DimensionPacker` that can pack data ID dictionaries 

375 into unique integers. 

376 

377 Parameters 

378 ---------- 

379 name : `str` 

380 Name of the packer, matching a key in the "packers" section of the 

381 dimension configuration. 

382 dataId : `DataCoordinate` 

383 Fully-expanded data ID that identfies the at least the "fixed" 

384 dimensions of the packer (i.e. those that are assumed/given, 

385 setting the space over which packed integer IDs are unique). 

386 ``dataId.hasRecords()`` must return `True`. 

387 """ 

388 return self._packers[name](self, dataId) 

389 

390 def getEncodeLength(self) -> int: 

391 """Return the size (in bytes) of the encoded size of `DimensionGraph` 

392 instances in this universe. 

393 

394 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

395 information. 

396 """ 

397 return math.ceil(len(self._dimensions)/8) 

398 

399 @classmethod 

400 def _unpickle(cls, version: int) -> DimensionUniverse: 

401 """Callable used for unpickling. 

402 

403 For internal use only. 

404 """ 

405 try: 

406 return cls._instances[version] 

407 except KeyError as err: 

408 raise pickle.UnpicklingError( 

409 f"DimensionUniverse with version '{version}' " 

410 f"not found. Note that DimensionUniverse objects are not " 

411 f"truly serialized; when using pickle to transfer them " 

412 f"between processes, an equivalent instance with the same " 

413 f"version must already exist in the receiving process." 

414 ) from err 

415 

416 def __reduce__(self) -> tuple: 

417 return (self._unpickle, (self._version,)) 

418 

419 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

420 # DimensionUniverse is recursively immutable; see note in @immutable 

421 # decorator. 

422 return self 

423 

424 # Class attributes below are shadowed by instance attributes, and are 

425 # present just to hold the docstrings for those instance attributes. 

426 

427 empty: DimensionGraph 

428 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

429 """ 

430 

431 commonSkyPix: SkyPixDimension 

432 """The special skypix dimension that is used to relate all other spatial 

433 dimensions in the `Registry` database (`SkyPixDimension`). 

434 """ 

435 

436 _cache: Dict[FrozenSet[str], DimensionGraph] 

437 

438 _dimensions: NamedValueAbstractSet[Dimension] 

439 

440 _elements: NamedValueAbstractSet[DimensionElement] 

441 

442 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

443 

444 _dimensionIndices: Dict[str, int] 

445 

446 _elementIndices: Dict[str, int] 

447 

448 _packers: Dict[str, DimensionPackerFactory] 

449 

450 _version: int