Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 36%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

126 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 ClassVar, 

30 Dict, 

31 FrozenSet, 

32 Iterable, 

33 List, 

34 Mapping, 

35 Optional, 

36 Set, 

37 TYPE_CHECKING, 

38 TypeVar, 

39 Union, 

40) 

41 

42from lsst.utils.classes import cached_getter, immutable 

43from ..config import Config 

44from ..named import NamedValueAbstractSet, NamedValueSet 

45from .._topology import TopologicalSpace, TopologicalFamily 

46from ._config import DimensionConfig 

47from ._elements import Dimension, DimensionElement 

48from ._graph import DimensionGraph 

49from ._governor import GovernorDimension 

50from ._database import DatabaseDimensionElement 

51from ._skypix import SkyPixDimension, SkyPixSystem 

52 

53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true

54 from ._coordinate import DataCoordinate 

55 from ._packer import DimensionPacker, DimensionPackerFactory 

56 from .construction import DimensionConstructionBuilder 

57 

58 

59E = TypeVar("E", bound=DimensionElement) 

60 

61 

62@immutable 

63class DimensionUniverse: 

64 """Self-consistent set of dimensions. 

65 

66 A parent class that represents a complete, self-consistent set of 

67 dimensions and their relationships. 

68 

69 `DimensionUniverse` is not a class-level singleton, but all instances are 

70 tracked in a singleton map keyed by the version number in the configuration 

71 they were loaded from. Because these universes are solely responsible for 

72 constructing `DimensionElement` instances, these are also indirectly 

73 tracked by that singleton as well. 

74 

75 Parameters 

76 ---------- 

77 config : `Config`, optional 

78 Configuration object from which dimension definitions can be extracted. 

79 Ignored if ``builder`` is provided, or if ``version`` is provided and 

80 an instance with that version already exists. 

81 version : `int`, optional 

82 Integer version for this `DimensionUniverse`. If not provided, a 

83 version will be obtained from ``builder`` or ``config``. 

84 builder : `DimensionConstructionBuilder`, optional 

85 Builder object used to initialize a new instance. Ignored if 

86 ``version`` is provided and an instance with that version already 

87 exists. Should not have had `~DimensionConstructionBuilder.finish` 

88 called; this will be called if needed by `DimensionUniverse`. 

89 """ 

90 

91 _instances: ClassVar[Dict[int, DimensionUniverse]] = {} 

92 """Singleton dictionary of all instances, keyed by version. 

93 

94 For internal use only. 

95 """ 

96 

97 def __new__( 

98 cls, 

99 config: Optional[Config] = None, *, 

100 version: Optional[int] = None, 

101 builder: Optional[DimensionConstructionBuilder] = None, 

102 ) -> DimensionUniverse: 

103 # Try to get a version first, to look for existing instances; try to 

104 # do as little work as possible at this stage. 

105 if version is None: 

106 if builder is None: 

107 config = DimensionConfig(config) 

108 version = config["version"] 

109 else: 

110 version = builder.version 

111 

112 # See if an equivalent instance already exists. 

113 self: Optional[DimensionUniverse] = cls._instances.get(version) 

114 if self is not None: 

115 return self 

116 

117 # Ensure we have a builder, building one from config if necessary. 

118 if builder is None: 

119 config = DimensionConfig(config) 

120 builder = config.makeBuilder() 

121 

122 # Delegate to the builder for most of the construction work. 

123 builder.finish() 

124 

125 # Create the universe instance and create core attributes, mostly 

126 # copying from builder. 

127 self = object.__new__(cls) 

128 assert self is not None 

129 self._cache = {} 

130 self._dimensions = builder.dimensions 

131 self._elements = builder.elements 

132 self._topology = builder.topology 

133 self._packers = builder.packers 

134 self.dimensionConfig = builder.config 

135 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

136 assert isinstance(commonSkyPix, SkyPixDimension) 

137 self.commonSkyPix = commonSkyPix 

138 

139 # Attach self to all elements. 

140 for element in self._elements: 

141 element.universe = self 

142 

143 # Add attribute for special subsets of the graph. 

144 self.empty = DimensionGraph(self, (), conform=False) 

145 

146 # Use the version number from the config as a key in the singleton 

147 # dict containing all instances; that will let us transfer dimension 

148 # objects between processes using pickle without actually going 

149 # through real initialization, as long as a universe with the same 

150 # version has already been constructed in the receiving process. 

151 self._version = version 

152 cls._instances[self._version] = self 

153 

154 # Build mappings from element to index. These are used for 

155 # topological-sort comparison operators in DimensionElement itself. 

156 self._elementIndices = { 

157 name: i for i, name in enumerate(self._elements.names) 

158 } 

159 # Same for dimension to index, sorted topologically across required 

160 # and implied. This is used for encode/decode. 

161 self._dimensionIndices = { 

162 name: i for i, name in enumerate(self._dimensions.names) 

163 } 

164 

165 return self 

166 

167 def __repr__(self) -> str: 

168 return f"DimensionUniverse({self._version})" 

169 

170 def __getitem__(self, name: str) -> DimensionElement: 

171 return self._elements[name] 

172 

173 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

174 """Return the `DimensionElement` with the given name or a default. 

175 

176 Parameters 

177 ---------- 

178 name : `str` 

179 Name of the element. 

180 default : `DimensionElement`, optional 

181 Element to return if the named one does not exist. Defaults to 

182 `None`. 

183 

184 Returns 

185 ------- 

186 element : `DimensionElement` 

187 The named element. 

188 """ 

189 return self._elements.get(name, default) 

190 

191 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

192 """Return a set of all static elements in this universe. 

193 

194 Non-static elements that are created as needed may also exist, but 

195 these are guaranteed to have no direct relationships to other elements 

196 (though they may have spatial or temporal relationships). 

197 

198 Returns 

199 ------- 

200 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

201 A frozen set of `DimensionElement` instances. 

202 """ 

203 return self._elements 

204 

205 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

206 """Return a set of all static dimensions in this universe. 

207 

208 Non-static dimensions that are created as needed may also exist, but 

209 these are guaranteed to have no direct relationships to other elements 

210 (though they may have spatial or temporal relationships). 

211 

212 Returns 

213 ------- 

214 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

215 A frozen set of `Dimension` instances. 

216 """ 

217 return self._dimensions 

218 

219 @cached_getter 

220 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

221 """Return a set of all `GovernorDimension` instances in this universe. 

222 

223 Returns 

224 ------- 

225 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

226 A frozen set of `GovernorDimension` instances. 

227 """ 

228 return NamedValueSet( 

229 d for d in self._dimensions if isinstance(d, GovernorDimension) 

230 ).freeze() 

231 

232 @cached_getter 

233 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

234 """Return set of all `DatabaseDimensionElement` instances in universe. 

235 

236 This does not include `GovernorDimension` instances, which are backed 

237 by the database but do not inherit from `DatabaseDimensionElement`. 

238 

239 Returns 

240 ------- 

241 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

242 A frozen set of `DatabaseDimensionElement` instances. 

243 """ 

244 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

245 

246 @property # type: ignore 

247 @cached_getter 

248 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

249 """All skypix systems known to this universe. 

250 

251 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

252 """ 

253 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL] 

254 if isinstance(family, SkyPixSystem)]).freeze() 

255 

256 def getElementIndex(self, name: str) -> int: 

257 """Return the position of the named dimension element. 

258 

259 The position is in this universe's sorting of all elements. 

260 

261 Parameters 

262 ---------- 

263 name : `str` 

264 Name of the element. 

265 

266 Returns 

267 ------- 

268 index : `int` 

269 Sorting index for this element. 

270 """ 

271 return self._elementIndices[name] 

272 

273 def getDimensionIndex(self, name: str) -> int: 

274 """Return the position of the named dimension. 

275 

276 This position is in this universe's sorting of all dimensions. 

277 

278 Parameters 

279 ---------- 

280 name : `str` 

281 Name of the dimension. 

282 

283 Returns 

284 ------- 

285 index : `int` 

286 Sorting index for this dimension. 

287 

288 Notes 

289 ----- 

290 The dimension sort order for a universe is consistent with the element 

291 order (all dimensions are elements), and either can be used to sort 

292 dimensions if used consistently. But there are also some contexts in 

293 which contiguous dimension-only indices are necessary or at least 

294 desirable. 

295 """ 

296 return self._dimensionIndices[name] 

297 

298 def expandDimensionNameSet(self, names: Set[str]) -> None: 

299 """Expand a set of dimension names in-place. 

300 

301 Includes recursive dependencies. 

302 

303 This is an advanced interface for cases where constructing a 

304 `DimensionGraph` (which also expands required dependencies) is 

305 impossible or undesirable. 

306 

307 Parameters 

308 ---------- 

309 names : `set` [ `str` ] 

310 A true `set` of dimension names, to be expanded in-place. 

311 """ 

312 # Keep iterating until the set of names stops growing. This is not as 

313 # efficient as it could be, but we work pretty hard cache 

314 # DimensionGraph instances to keep actual construction rare, so that 

315 # shouldn't matter. 

316 oldSize = len(names) 

317 while True: 

318 # iterate over a temporary copy so we can modify the original 

319 for name in tuple(names): 

320 names.update(self._dimensions[name].required.names) 

321 names.update(self._dimensions[name].implied.names) 

322 if oldSize == len(names): 

323 break 

324 else: 

325 oldSize = len(names) 

326 

327 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

328 """Construct graph from iterable. 

329 

330 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

331 of `Dimension` instances and string names thereof. 

332 

333 Constructing `DimensionGraph` directly from names or dimension 

334 instances is slightly more efficient when it is known in advance that 

335 the iterable is not heterogenous. 

336 

337 Parameters 

338 ---------- 

339 iterable: iterable of `Dimension` or `str` 

340 Dimensions that must be included in the returned graph (their 

341 dependencies will be as well). 

342 

343 Returns 

344 ------- 

345 graph : `DimensionGraph` 

346 A `DimensionGraph` instance containing all given dimensions. 

347 """ 

348 names = set() 

349 for item in iterable: 

350 try: 

351 names.add(item.name) # type: ignore 

352 except AttributeError: 

353 names.add(item) 

354 return DimensionGraph(universe=self, names=names) 

355 

356 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

357 """Return a sorted version of the given iterable of dimension elements. 

358 

359 The universe's sort order is topological (an element's dependencies 

360 precede it), with an unspecified (but deterministic) approach to 

361 breaking ties. 

362 

363 Parameters 

364 ---------- 

365 elements : iterable of `DimensionElement`. 

366 Elements to be sorted. 

367 reverse : `bool`, optional 

368 If `True`, sort in the opposite order. 

369 

370 Returns 

371 ------- 

372 sorted : `list` of `DimensionElement` 

373 A sorted list containing the same elements that were given. 

374 """ 

375 s = set(elements) 

376 result = [element for element in self._elements if element in s or element.name in s] 

377 if reverse: 

378 result.reverse() 

379 # mypy thinks this can return DimensionElements even if all the user 

380 # passed it was Dimensions; we know better. 

381 return result # type: ignore 

382 

383 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

384 """Make a dimension packer. 

385 

386 Constructs a `DimensionPacker` that can pack data ID dictionaries 

387 into unique integers. 

388 

389 Parameters 

390 ---------- 

391 name : `str` 

392 Name of the packer, matching a key in the "packers" section of the 

393 dimension configuration. 

394 dataId : `DataCoordinate` 

395 Fully-expanded data ID that identifies the at least the "fixed" 

396 dimensions of the packer (i.e. those that are assumed/given, 

397 setting the space over which packed integer IDs are unique). 

398 ``dataId.hasRecords()`` must return `True`. 

399 """ 

400 return self._packers[name](self, dataId) 

401 

402 def getEncodeLength(self) -> int: 

403 """Return encoded size of graph. 

404 

405 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

406 instances in this universe. 

407 

408 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

409 information. 

410 """ 

411 return math.ceil(len(self._dimensions)/8) 

412 

413 @classmethod 

414 def _unpickle(cls, version: int) -> DimensionUniverse: 

415 """Return an unpickled dimension universe. 

416 

417 Callable used for unpickling. 

418 

419 For internal use only. 

420 """ 

421 try: 

422 return cls._instances[version] 

423 except KeyError as err: 

424 raise pickle.UnpicklingError( 

425 f"DimensionUniverse with version '{version}' " 

426 f"not found. Note that DimensionUniverse objects are not " 

427 f"truly serialized; when using pickle to transfer them " 

428 f"between processes, an equivalent instance with the same " 

429 f"version must already exist in the receiving process." 

430 ) from err 

431 

432 def __reduce__(self) -> tuple: 

433 return (self._unpickle, (self._version,)) 

434 

435 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

436 # DimensionUniverse is recursively immutable; see note in @immutable 

437 # decorator. 

438 return self 

439 

440 # Class attributes below are shadowed by instance attributes, and are 

441 # present just to hold the docstrings for those instance attributes. 

442 

443 empty: DimensionGraph 

444 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

445 """ 

446 

447 commonSkyPix: SkyPixDimension 

448 """The special skypix dimension that is used to relate all other spatial 

449 dimensions in the `Registry` database (`SkyPixDimension`). 

450 """ 

451 

452 dimensionConfig: DimensionConfig 

453 """The configuration used to create this Universe (`DimensionConfig`).""" 

454 

455 _cache: Dict[FrozenSet[str], DimensionGraph] 

456 

457 _dimensions: NamedValueAbstractSet[Dimension] 

458 

459 _elements: NamedValueAbstractSet[DimensionElement] 

460 

461 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

462 

463 _dimensionIndices: Dict[str, int] 

464 

465 _elementIndices: Dict[str, int] 

466 

467 _packers: Dict[str, DimensionPackerFactory] 

468 

469 _version: int