Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 36%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

126 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 TYPE_CHECKING, 

30 ClassVar, 

31 Dict, 

32 FrozenSet, 

33 Iterable, 

34 List, 

35 Mapping, 

36 Optional, 

37 Set, 

38 TypeVar, 

39 Union, 

40) 

41 

42from lsst.utils.classes import cached_getter, immutable 

43 

44from .._topology import TopologicalFamily, TopologicalSpace 

45from ..config import Config 

46from ..named import NamedValueAbstractSet, NamedValueSet 

47from ._config import DimensionConfig 

48from ._database import DatabaseDimensionElement 

49from ._elements import Dimension, DimensionElement 

50from ._governor import GovernorDimension 

51from ._graph import DimensionGraph 

52from ._skypix import SkyPixDimension, SkyPixSystem 

53 

54if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 54 ↛ 55line 54 didn't jump to line 55, because the condition on line 54 was never true

55 from ._coordinate import DataCoordinate 

56 from ._packer import DimensionPacker, DimensionPackerFactory 

57 from .construction import DimensionConstructionBuilder 

58 

59 

60E = TypeVar("E", bound=DimensionElement) 

61 

62 

63@immutable 

64class DimensionUniverse: 

65 """Self-consistent set of dimensions. 

66 

67 A parent class that represents a complete, self-consistent set of 

68 dimensions and their relationships. 

69 

70 `DimensionUniverse` is not a class-level singleton, but all instances are 

71 tracked in a singleton map keyed by the version number in the configuration 

72 they were loaded from. Because these universes are solely responsible for 

73 constructing `DimensionElement` instances, these are also indirectly 

74 tracked by that singleton as well. 

75 

76 Parameters 

77 ---------- 

78 config : `Config`, optional 

79 Configuration object from which dimension definitions can be extracted. 

80 Ignored if ``builder`` is provided, or if ``version`` is provided and 

81 an instance with that version already exists. 

82 version : `int`, optional 

83 Integer version for this `DimensionUniverse`. If not provided, a 

84 version will be obtained from ``builder`` or ``config``. 

85 builder : `DimensionConstructionBuilder`, optional 

86 Builder object used to initialize a new instance. Ignored if 

87 ``version`` is provided and an instance with that version already 

88 exists. Should not have had `~DimensionConstructionBuilder.finish` 

89 called; this will be called if needed by `DimensionUniverse`. 

90 """ 

91 

92 _instances: ClassVar[Dict[int, DimensionUniverse]] = {} 

93 """Singleton dictionary of all instances, keyed by version. 

94 

95 For internal use only. 

96 """ 

97 

98 def __new__( 

99 cls, 

100 config: Optional[Config] = None, 

101 *, 

102 version: Optional[int] = None, 

103 builder: Optional[DimensionConstructionBuilder] = None, 

104 ) -> DimensionUniverse: 

105 # Try to get a version first, to look for existing instances; try to 

106 # do as little work as possible at this stage. 

107 if version is None: 

108 if builder is None: 

109 config = DimensionConfig(config) 

110 version = config["version"] 

111 else: 

112 version = builder.version 

113 

114 # See if an equivalent instance already exists. 

115 self: Optional[DimensionUniverse] = cls._instances.get(version) 

116 if self is not None: 

117 return self 

118 

119 # Ensure we have a builder, building one from config if necessary. 

120 if builder is None: 

121 config = DimensionConfig(config) 

122 builder = config.makeBuilder() 

123 

124 # Delegate to the builder for most of the construction work. 

125 builder.finish() 

126 

127 # Create the universe instance and create core attributes, mostly 

128 # copying from builder. 

129 self = object.__new__(cls) 

130 assert self is not None 

131 self._cache = {} 

132 self._dimensions = builder.dimensions 

133 self._elements = builder.elements 

134 self._topology = builder.topology 

135 self._packers = builder.packers 

136 self.dimensionConfig = builder.config 

137 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

138 assert isinstance(commonSkyPix, SkyPixDimension) 

139 self.commonSkyPix = commonSkyPix 

140 

141 # Attach self to all elements. 

142 for element in self._elements: 

143 element.universe = self 

144 

145 # Add attribute for special subsets of the graph. 

146 self.empty = DimensionGraph(self, (), conform=False) 

147 

148 # Use the version number from the config as a key in the singleton 

149 # dict containing all instances; that will let us transfer dimension 

150 # objects between processes using pickle without actually going 

151 # through real initialization, as long as a universe with the same 

152 # version has already been constructed in the receiving process. 

153 self._version = version 

154 cls._instances[self._version] = self 

155 

156 # Build mappings from element to index. These are used for 

157 # topological-sort comparison operators in DimensionElement itself. 

158 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

159 # Same for dimension to index, sorted topologically across required 

160 # and implied. This is used for encode/decode. 

161 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

162 

163 return self 

164 

165 def __repr__(self) -> str: 

166 return f"DimensionUniverse({self._version})" 

167 

168 def __getitem__(self, name: str) -> DimensionElement: 

169 return self._elements[name] 

170 

171 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

172 """Return the `DimensionElement` with the given name or a default. 

173 

174 Parameters 

175 ---------- 

176 name : `str` 

177 Name of the element. 

178 default : `DimensionElement`, optional 

179 Element to return if the named one does not exist. Defaults to 

180 `None`. 

181 

182 Returns 

183 ------- 

184 element : `DimensionElement` 

185 The named element. 

186 """ 

187 return self._elements.get(name, default) 

188 

189 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

190 """Return a set of all static elements in this universe. 

191 

192 Non-static elements that are created as needed may also exist, but 

193 these are guaranteed to have no direct relationships to other elements 

194 (though they may have spatial or temporal relationships). 

195 

196 Returns 

197 ------- 

198 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

199 A frozen set of `DimensionElement` instances. 

200 """ 

201 return self._elements 

202 

203 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

204 """Return a set of all static dimensions in this universe. 

205 

206 Non-static dimensions that are created as needed may also exist, but 

207 these are guaranteed to have no direct relationships to other elements 

208 (though they may have spatial or temporal relationships). 

209 

210 Returns 

211 ------- 

212 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

213 A frozen set of `Dimension` instances. 

214 """ 

215 return self._dimensions 

216 

217 @cached_getter 

218 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

219 """Return a set of all `GovernorDimension` instances in this universe. 

220 

221 Returns 

222 ------- 

223 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

224 A frozen set of `GovernorDimension` instances. 

225 """ 

226 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

227 

228 @cached_getter 

229 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

230 """Return set of all `DatabaseDimensionElement` instances in universe. 

231 

232 This does not include `GovernorDimension` instances, which are backed 

233 by the database but do not inherit from `DatabaseDimensionElement`. 

234 

235 Returns 

236 ------- 

237 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

238 A frozen set of `DatabaseDimensionElement` instances. 

239 """ 

240 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

241 

242 @property # type: ignore 

243 @cached_getter 

244 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

245 """All skypix systems known to this universe. 

246 

247 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

248 """ 

249 return NamedValueSet( 

250 [ 

251 family 

252 for family in self._topology[TopologicalSpace.SPATIAL] 

253 if isinstance(family, SkyPixSystem) 

254 ] 

255 ).freeze() 

256 

257 def getElementIndex(self, name: str) -> int: 

258 """Return the position of the named dimension element. 

259 

260 The position is in this universe's sorting of all elements. 

261 

262 Parameters 

263 ---------- 

264 name : `str` 

265 Name of the element. 

266 

267 Returns 

268 ------- 

269 index : `int` 

270 Sorting index for this element. 

271 """ 

272 return self._elementIndices[name] 

273 

274 def getDimensionIndex(self, name: str) -> int: 

275 """Return the position of the named dimension. 

276 

277 This position is in this universe's sorting of all dimensions. 

278 

279 Parameters 

280 ---------- 

281 name : `str` 

282 Name of the dimension. 

283 

284 Returns 

285 ------- 

286 index : `int` 

287 Sorting index for this dimension. 

288 

289 Notes 

290 ----- 

291 The dimension sort order for a universe is consistent with the element 

292 order (all dimensions are elements), and either can be used to sort 

293 dimensions if used consistently. But there are also some contexts in 

294 which contiguous dimension-only indices are necessary or at least 

295 desirable. 

296 """ 

297 return self._dimensionIndices[name] 

298 

299 def expandDimensionNameSet(self, names: Set[str]) -> None: 

300 """Expand a set of dimension names in-place. 

301 

302 Includes recursive dependencies. 

303 

304 This is an advanced interface for cases where constructing a 

305 `DimensionGraph` (which also expands required dependencies) is 

306 impossible or undesirable. 

307 

308 Parameters 

309 ---------- 

310 names : `set` [ `str` ] 

311 A true `set` of dimension names, to be expanded in-place. 

312 """ 

313 # Keep iterating until the set of names stops growing. This is not as 

314 # efficient as it could be, but we work pretty hard cache 

315 # DimensionGraph instances to keep actual construction rare, so that 

316 # shouldn't matter. 

317 oldSize = len(names) 

318 while True: 

319 # iterate over a temporary copy so we can modify the original 

320 for name in tuple(names): 

321 names.update(self._dimensions[name].required.names) 

322 names.update(self._dimensions[name].implied.names) 

323 if oldSize == len(names): 

324 break 

325 else: 

326 oldSize = len(names) 

327 

328 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

329 """Construct graph from iterable. 

330 

331 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

332 of `Dimension` instances and string names thereof. 

333 

334 Constructing `DimensionGraph` directly from names or dimension 

335 instances is slightly more efficient when it is known in advance that 

336 the iterable is not heterogenous. 

337 

338 Parameters 

339 ---------- 

340 iterable: iterable of `Dimension` or `str` 

341 Dimensions that must be included in the returned graph (their 

342 dependencies will be as well). 

343 

344 Returns 

345 ------- 

346 graph : `DimensionGraph` 

347 A `DimensionGraph` instance containing all given dimensions. 

348 """ 

349 names = set() 

350 for item in iterable: 

351 try: 

352 names.add(item.name) # type: ignore 

353 except AttributeError: 

354 names.add(item) 

355 return DimensionGraph(universe=self, names=names) 

356 

357 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

358 """Return a sorted version of the given iterable of dimension elements. 

359 

360 The universe's sort order is topological (an element's dependencies 

361 precede it), with an unspecified (but deterministic) approach to 

362 breaking ties. 

363 

364 Parameters 

365 ---------- 

366 elements : iterable of `DimensionElement`. 

367 Elements to be sorted. 

368 reverse : `bool`, optional 

369 If `True`, sort in the opposite order. 

370 

371 Returns 

372 ------- 

373 sorted : `list` of `DimensionElement` 

374 A sorted list containing the same elements that were given. 

375 """ 

376 s = set(elements) 

377 result = [element for element in self._elements if element in s or element.name in s] 

378 if reverse: 

379 result.reverse() 

380 # mypy thinks this can return DimensionElements even if all the user 

381 # passed it was Dimensions; we know better. 

382 return result # type: ignore 

383 

384 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

385 """Make a dimension packer. 

386 

387 Constructs a `DimensionPacker` that can pack data ID dictionaries 

388 into unique integers. 

389 

390 Parameters 

391 ---------- 

392 name : `str` 

393 Name of the packer, matching a key in the "packers" section of the 

394 dimension configuration. 

395 dataId : `DataCoordinate` 

396 Fully-expanded data ID that identifies the at least the "fixed" 

397 dimensions of the packer (i.e. those that are assumed/given, 

398 setting the space over which packed integer IDs are unique). 

399 ``dataId.hasRecords()`` must return `True`. 

400 """ 

401 return self._packers[name](self, dataId) 

402 

403 def getEncodeLength(self) -> int: 

404 """Return encoded size of graph. 

405 

406 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

407 instances in this universe. 

408 

409 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

410 information. 

411 """ 

412 return math.ceil(len(self._dimensions) / 8) 

413 

414 @classmethod 

415 def _unpickle(cls, version: int) -> DimensionUniverse: 

416 """Return an unpickled dimension universe. 

417 

418 Callable used for unpickling. 

419 

420 For internal use only. 

421 """ 

422 try: 

423 return cls._instances[version] 

424 except KeyError as err: 

425 raise pickle.UnpicklingError( 

426 f"DimensionUniverse with version '{version}' " 

427 f"not found. Note that DimensionUniverse objects are not " 

428 f"truly serialized; when using pickle to transfer them " 

429 f"between processes, an equivalent instance with the same " 

430 f"version must already exist in the receiving process." 

431 ) from err 

432 

433 def __reduce__(self) -> tuple: 

434 return (self._unpickle, (self._version,)) 

435 

436 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

437 # DimensionUniverse is recursively immutable; see note in @immutable 

438 # decorator. 

439 return self 

440 

441 # Class attributes below are shadowed by instance attributes, and are 

442 # present just to hold the docstrings for those instance attributes. 

443 

444 empty: DimensionGraph 

445 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

446 """ 

447 

448 commonSkyPix: SkyPixDimension 

449 """The special skypix dimension that is used to relate all other spatial 

450 dimensions in the `Registry` database (`SkyPixDimension`). 

451 """ 

452 

453 dimensionConfig: DimensionConfig 

454 """The configuration used to create this Universe (`DimensionConfig`).""" 

455 

456 _cache: Dict[FrozenSet[str], DimensionGraph] 

457 

458 _dimensions: NamedValueAbstractSet[Dimension] 

459 

460 _elements: NamedValueAbstractSet[DimensionElement] 

461 

462 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

463 

464 _dimensionIndices: Dict[str, int] 

465 

466 _elementIndices: Dict[str, int] 

467 

468 _packers: Dict[str, DimensionPackerFactory] 

469 

470 _version: int