Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 ClassVar, 

30 Dict, 

31 FrozenSet, 

32 Iterable, 

33 List, 

34 Mapping, 

35 Optional, 

36 Set, 

37 TYPE_CHECKING, 

38 TypeVar, 

39 Union, 

40) 

41 

42from ..config import Config 

43from ..named import NamedValueAbstractSet, NamedValueSet 

44from .._topology import TopologicalSpace, TopologicalFamily 

45from ..utils import cached_getter, immutable 

46from ._config import DimensionConfig 

47from ._elements import Dimension, DimensionElement 

48from ._graph import DimensionGraph 

49from ._governor import GovernorDimension 

50from ._database import DatabaseDimensionElement 

51from ._skypix import SkyPixDimension, SkyPixSystem 

52 

53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true

54 from ._coordinate import DataCoordinate 

55 from ._packer import DimensionPacker, DimensionPackerFactory 

56 from .construction import DimensionConstructionBuilder 

57 

58 

59E = TypeVar("E", bound=DimensionElement) 

60 

61 

62@immutable 

63class DimensionUniverse: 

64 """Self-consistent set of dimensions. 

65 

66 A parent class that represents a complete, self-consistent set of 

67 dimensions and their relationships. 

68 

69 `DimensionUniverse` is not a class-level singleton, but all instances are 

70 tracked in a singleton map keyed by the version number in the configuration 

71 they were loaded from. Because these universes are solely responsible for 

72 constructing `DimensionElement` instances, these are also indirectly 

73 tracked by that singleton as well. 

74 

75 Parameters 

76 ---------- 

77 config : `Config`, optional 

78 Configuration object from which dimension definitions can be extracted. 

79 Ignored if ``builder`` is provided, or if ``version`` is provided and 

80 an instance with that version already exists. 

81 version : `int`, optional 

82 Integer version for this `DimensionUniverse`. If not provided, a 

83 version will be obtained from ``builder`` or ``config``. 

84 builder : `DimensionConstructionBuilder`, optional 

85 Builder object used to initialize a new instance. Ignored if 

86 ``version`` is provided and an instance with that version already 

87 exists. Should not have had `~DimensionConstructionBuilder.finish` 

88 called; this will be called if needed by `DimensionUniverse`. 

89 """ 

90 

91 _instances: ClassVar[Dict[int, DimensionUniverse]] = {} 

92 """Singleton dictionary of all instances, keyed by version. 

93 

94 For internal use only. 

95 """ 

96 

97 def __new__( 

98 cls, 

99 config: Optional[Config] = None, *, 

100 version: Optional[int] = None, 

101 builder: Optional[DimensionConstructionBuilder] = None, 

102 ) -> DimensionUniverse: 

103 # Try to get a version first, to look for existing instances; try to 

104 # do as little work as possible at this stage. 

105 if version is None: 

106 if builder is None: 

107 config = DimensionConfig(config) 

108 version = config["version"] 

109 else: 

110 version = builder.version 

111 

112 # See if an equivalent instance already exists. 

113 self: Optional[DimensionUniverse] = cls._instances.get(version) 

114 if self is not None: 

115 return self 

116 

117 # Ensure we have a builder, building one from config if necessary. 

118 if builder is None: 

119 config = DimensionConfig(config) 

120 builder = config.makeBuilder() 

121 

122 # Delegate to the builder for most of the construction work. 

123 builder.finish() 

124 

125 # Create the universe instance and create core attributes, mostly 

126 # copying from builder. 

127 self = object.__new__(cls) 

128 assert self is not None 

129 self._cache = {} 

130 self._dimensions = builder.dimensions 

131 self._elements = builder.elements 

132 self._topology = builder.topology 

133 self._packers = builder.packers 

134 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

135 assert isinstance(commonSkyPix, SkyPixDimension) 

136 self.commonSkyPix = commonSkyPix 

137 

138 # Attach self to all elements. 

139 for element in self._elements: 

140 element.universe = self 

141 

142 # Add attribute for special subsets of the graph. 

143 self.empty = DimensionGraph(self, (), conform=False) 

144 

145 # Use the version number from the config as a key in the singleton 

146 # dict containing all instances; that will let us transfer dimension 

147 # objects between processes using pickle without actually going 

148 # through real initialization, as long as a universe with the same 

149 # version has already been constructed in the receiving process. 

150 self._version = version 

151 cls._instances[self._version] = self 

152 

153 # Build mappings from element to index. These are used for 

154 # topological-sort comparison operators in DimensionElement itself. 

155 self._elementIndices = { 

156 name: i for i, name in enumerate(self._elements.names) 

157 } 

158 # Same for dimension to index, sorted topologically across required 

159 # and implied. This is used for encode/decode. 

160 self._dimensionIndices = { 

161 name: i for i, name in enumerate(self._dimensions.names) 

162 } 

163 

164 return self 

165 

166 def __repr__(self) -> str: 

167 return f"DimensionUniverse({self._version})" 

168 

169 def __getitem__(self, name: str) -> DimensionElement: 

170 return self._elements[name] 

171 

172 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

173 """Return the `DimensionElement` with the given name or a default. 

174 

175 Parameters 

176 ---------- 

177 name : `str` 

178 Name of the element. 

179 default : `DimensionElement`, optional 

180 Element to return if the named one does not exist. Defaults to 

181 `None`. 

182 

183 Returns 

184 ------- 

185 element : `DimensionElement` 

186 The named element. 

187 """ 

188 return self._elements.get(name, default) 

189 

190 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

191 """Return a set of all static elements in this universe. 

192 

193 Non-static elements that are created as needed may also exist, but 

194 these are guaranteed to have no direct relationships to other elements 

195 (though they may have spatial or temporal relationships). 

196 

197 Returns 

198 ------- 

199 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

200 A frozen set of `DimensionElement` instances. 

201 """ 

202 return self._elements 

203 

204 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

205 """Return a set of all static dimensions in this universe. 

206 

207 Non-static dimensions that are created as needed may also exist, but 

208 these are guaranteed to have no direct relationships to other elements 

209 (though they may have spatial or temporal relationships). 

210 

211 Returns 

212 ------- 

213 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

214 A frozen set of `Dimension` instances. 

215 """ 

216 return self._dimensions 

217 

218 @cached_getter 

219 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

220 """Return a set of all `GovernorDimension` instances in this universe. 

221 

222 Returns 

223 ------- 

224 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

225 A frozen set of `GovernorDimension` instances. 

226 """ 

227 return NamedValueSet( 

228 d for d in self._dimensions if isinstance(d, GovernorDimension) 

229 ).freeze() 

230 

231 @cached_getter 

232 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

233 """Return set of all `DatabaseDimensionElement` instances in universe. 

234 

235 This does not include `GovernorDimension` instances, which are backed 

236 by the database but do not inherit from `DatabaseDimensionElement`. 

237 

238 Returns 

239 ------- 

240 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

241 A frozen set of `DatabaseDimensionElement` instances. 

242 """ 

243 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

244 

245 @property # type: ignore 

246 @cached_getter 

247 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

248 """All skypix systems known to this universe. 

249 

250 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

251 """ 

252 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL] 

253 if isinstance(family, SkyPixSystem)]).freeze() 

254 

255 def getElementIndex(self, name: str) -> int: 

256 """Return the position of the named dimension element. 

257 

258 The position is in this universe's sorting of all elements. 

259 

260 Parameters 

261 ---------- 

262 name : `str` 

263 Name of the element. 

264 

265 Returns 

266 ------- 

267 index : `int` 

268 Sorting index for this element. 

269 """ 

270 return self._elementIndices[name] 

271 

272 def getDimensionIndex(self, name: str) -> int: 

273 """Return the position of the named dimension. 

274 

275 This position is in this universe's sorting of all dimensions. 

276 

277 Parameters 

278 ---------- 

279 name : `str` 

280 Name of the dimension. 

281 

282 Returns 

283 ------- 

284 index : `int` 

285 Sorting index for this dimension. 

286 

287 Notes 

288 ----- 

289 The dimension sort order for a universe is consistent with the element 

290 order (all dimensions are elements), and either can be used to sort 

291 dimensions if used consistently. But there are also some contexts in 

292 which contiguous dimension-only indices are necessary or at least 

293 desirable. 

294 """ 

295 return self._dimensionIndices[name] 

296 

297 def expandDimensionNameSet(self, names: Set[str]) -> None: 

298 """Expand a set of dimension names in-place. 

299 

300 Includes recursive dependencies. 

301 

302 This is an advanced interface for cases where constructing a 

303 `DimensionGraph` (which also expands required dependencies) is 

304 impossible or undesirable. 

305 

306 Parameters 

307 ---------- 

308 names : `set` [ `str` ] 

309 A true `set` of dimension names, to be expanded in-place. 

310 """ 

311 # Keep iterating until the set of names stops growing. This is not as 

312 # efficient as it could be, but we work pretty hard cache 

313 # DimensionGraph instances to keep actual construction rare, so that 

314 # shouldn't matter. 

315 oldSize = len(names) 

316 while True: 

317 # iterate over a temporary copy so we can modify the original 

318 for name in tuple(names): 

319 names.update(self._dimensions[name].required.names) 

320 names.update(self._dimensions[name].implied.names) 

321 if oldSize == len(names): 

322 break 

323 else: 

324 oldSize = len(names) 

325 

326 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

327 """Construct graph from iterable. 

328 

329 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

330 of `Dimension` instances and string names thereof. 

331 

332 Constructing `DimensionGraph` directly from names or dimension 

333 instances is slightly more efficient when it is known in advance that 

334 the iterable is not heterogenous. 

335 

336 Parameters 

337 ---------- 

338 iterable: iterable of `Dimension` or `str` 

339 Dimensions that must be included in the returned graph (their 

340 dependencies will be as well). 

341 

342 Returns 

343 ------- 

344 graph : `DimensionGraph` 

345 A `DimensionGraph` instance containing all given dimensions. 

346 """ 

347 names = set() 

348 for item in iterable: 

349 try: 

350 names.add(item.name) # type: ignore 

351 except AttributeError: 

352 names.add(item) 

353 return DimensionGraph(universe=self, names=names) 

354 

355 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

356 """Return a sorted version of the given iterable of dimension elements. 

357 

358 The universe's sort order is topological (an element's dependencies 

359 precede it), with an unspecified (but deterministic) approach to 

360 breaking ties. 

361 

362 Parameters 

363 ---------- 

364 elements : iterable of `DimensionElement`. 

365 Elements to be sorted. 

366 reverse : `bool`, optional 

367 If `True`, sort in the opposite order. 

368 

369 Returns 

370 ------- 

371 sorted : `list` of `DimensionElement` 

372 A sorted list containing the same elements that were given. 

373 """ 

374 s = set(elements) 

375 result = [element for element in self._elements if element in s or element.name in s] 

376 if reverse: 

377 result.reverse() 

378 # mypy thinks this can return DimensionElements even if all the user 

379 # passed it was Dimensions; we know better. 

380 return result # type: ignore 

381 

382 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

383 """Make a dimension packer. 

384 

385 Constructs a `DimensionPacker` that can pack data ID dictionaries 

386 into unique integers. 

387 

388 Parameters 

389 ---------- 

390 name : `str` 

391 Name of the packer, matching a key in the "packers" section of the 

392 dimension configuration. 

393 dataId : `DataCoordinate` 

394 Fully-expanded data ID that identfies the at least the "fixed" 

395 dimensions of the packer (i.e. those that are assumed/given, 

396 setting the space over which packed integer IDs are unique). 

397 ``dataId.hasRecords()`` must return `True`. 

398 """ 

399 return self._packers[name](self, dataId) 

400 

401 def getEncodeLength(self) -> int: 

402 """Return encoded size of graph. 

403 

404 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

405 instances in this universe. 

406 

407 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

408 information. 

409 """ 

410 return math.ceil(len(self._dimensions)/8) 

411 

412 @classmethod 

413 def _unpickle(cls, version: int) -> DimensionUniverse: 

414 """Return an unpickled dimension universe. 

415 

416 Callable used for unpickling. 

417 

418 For internal use only. 

419 """ 

420 try: 

421 return cls._instances[version] 

422 except KeyError as err: 

423 raise pickle.UnpicklingError( 

424 f"DimensionUniverse with version '{version}' " 

425 f"not found. Note that DimensionUniverse objects are not " 

426 f"truly serialized; when using pickle to transfer them " 

427 f"between processes, an equivalent instance with the same " 

428 f"version must already exist in the receiving process." 

429 ) from err 

430 

431 def __reduce__(self) -> tuple: 

432 return (self._unpickle, (self._version,)) 

433 

434 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

435 # DimensionUniverse is recursively immutable; see note in @immutable 

436 # decorator. 

437 return self 

438 

439 # Class attributes below are shadowed by instance attributes, and are 

440 # present just to hold the docstrings for those instance attributes. 

441 

442 empty: DimensionGraph 

443 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

444 """ 

445 

446 commonSkyPix: SkyPixDimension 

447 """The special skypix dimension that is used to relate all other spatial 

448 dimensions in the `Registry` database (`SkyPixDimension`). 

449 """ 

450 

451 _cache: Dict[FrozenSet[str], DimensionGraph] 

452 

453 _dimensions: NamedValueAbstractSet[Dimension] 

454 

455 _elements: NamedValueAbstractSet[DimensionElement] 

456 

457 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

458 

459 _dimensionIndices: Dict[str, int] 

460 

461 _elementIndices: Dict[str, int] 

462 

463 _packers: Dict[str, DimensionPackerFactory] 

464 

465 _version: int