Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 33%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

137 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 TYPE_CHECKING, 

30 ClassVar, 

31 Dict, 

32 FrozenSet, 

33 Iterable, 

34 List, 

35 Mapping, 

36 Optional, 

37 Set, 

38 Tuple, 

39 TypeVar, 

40 Union, 

41) 

42 

43from lsst.utils.classes import cached_getter, immutable 

44 

45from .._topology import TopologicalFamily, TopologicalSpace 

46from ..config import Config 

47from ..named import NamedValueAbstractSet, NamedValueSet 

48from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

49from ._database import DatabaseDimensionElement 

50from ._elements import Dimension, DimensionElement 

51from ._governor import GovernorDimension 

52from ._graph import DimensionGraph 

53from ._skypix import SkyPixDimension, SkyPixSystem 

54 

55if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 55 ↛ 56line 55 didn't jump to line 56, because the condition on line 55 was never true

56 from ._coordinate import DataCoordinate 

57 from ._packer import DimensionPacker, DimensionPackerFactory 

58 from .construction import DimensionConstructionBuilder 

59 

60 

61E = TypeVar("E", bound=DimensionElement) 

62 

63 

64@immutable 

65class DimensionUniverse: 

66 """Self-consistent set of dimensions. 

67 

68 A parent class that represents a complete, self-consistent set of 

69 dimensions and their relationships. 

70 

71 `DimensionUniverse` is not a class-level singleton, but all instances are 

72 tracked in a singleton map keyed by the version number and namespace 

73 in the configuration they were loaded from. Because these universes 

74 are solely responsible for constructing `DimensionElement` instances, 

75 these are also indirectly tracked by that singleton as well. 

76 

77 Parameters 

78 ---------- 

79 config : `Config`, optional 

80 Configuration object from which dimension definitions can be extracted. 

81 Ignored if ``builder`` is provided, or if ``version`` is provided and 

82 an instance with that version already exists. 

83 version : `int`, optional 

84 Integer version for this `DimensionUniverse`. If not provided, a 

85 version will be obtained from ``builder`` or ``config``. 

86 namespace : `str`, optional 

87 Namespace of this `DimensionUniverse`, combined with the version 

88 to provide universe safety for registries that use different 

89 dimension definitions. 

90 builder : `DimensionConstructionBuilder`, optional 

91 Builder object used to initialize a new instance. Ignored if 

92 ``version`` is provided and an instance with that version already 

93 exists. Should not have had `~DimensionConstructionBuilder.finish` 

94 called; this will be called if needed by `DimensionUniverse`. 

95 """ 

96 

97 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {} 

98 """Singleton dictionary of all instances, keyed by version. 

99 

100 For internal use only. 

101 """ 

102 

103 def __new__( 

104 cls, 

105 config: Optional[Config] = None, 

106 *, 

107 version: Optional[int] = None, 

108 namespace: Optional[str] = None, 

109 builder: Optional[DimensionConstructionBuilder] = None, 

110 ) -> DimensionUniverse: 

111 # Try to get a version first, to look for existing instances; try to 

112 # do as little work as possible at this stage. 

113 if version is None: 

114 if builder is None: 

115 config = DimensionConfig(config) 

116 version = config["version"] 

117 else: 

118 version = builder.version 

119 

120 # Then a namespace. 

121 if namespace is None: 

122 if builder is None: 

123 config = DimensionConfig(config) 

124 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

125 else: 

126 namespace = builder.namespace 

127 # if still None use the default 

128 if namespace is None: 

129 namespace = _DEFAULT_NAMESPACE 

130 

131 # See if an equivalent instance already exists. 

132 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace)) 

133 if self is not None: 

134 return self 

135 

136 # Ensure we have a builder, building one from config if necessary. 

137 if builder is None: 

138 config = DimensionConfig(config) 

139 builder = config.makeBuilder() 

140 

141 # Delegate to the builder for most of the construction work. 

142 builder.finish() 

143 

144 # Create the universe instance and create core attributes, mostly 

145 # copying from builder. 

146 self = object.__new__(cls) 

147 assert self is not None 

148 self._cache = {} 

149 self._dimensions = builder.dimensions 

150 self._elements = builder.elements 

151 self._topology = builder.topology 

152 self._packers = builder.packers 

153 self.dimensionConfig = builder.config 

154 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

155 assert isinstance(commonSkyPix, SkyPixDimension) 

156 self.commonSkyPix = commonSkyPix 

157 

158 # Attach self to all elements. 

159 for element in self._elements: 

160 element.universe = self 

161 

162 # Add attribute for special subsets of the graph. 

163 self.empty = DimensionGraph(self, (), conform=False) 

164 

165 # Use the version number and namespace from the config as a key in 

166 # the singleton dict containing all instances; that will let us 

167 # transfer dimension objects between processes using pickle without 

168 # actually going through real initialization, as long as a universe 

169 # with the same version and namespace has already been constructed in 

170 # the receiving process. 

171 self._version = version 

172 self._namespace = namespace 

173 cls._instances[self._version, self._namespace] = self 

174 

175 # Build mappings from element to index. These are used for 

176 # topological-sort comparison operators in DimensionElement itself. 

177 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

178 # Same for dimension to index, sorted topologically across required 

179 # and implied. This is used for encode/decode. 

180 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

181 

182 return self 

183 

184 def __repr__(self) -> str: 

185 return f"DimensionUniverse({self._version}, {self._namespace})" 

186 

187 def __getitem__(self, name: str) -> DimensionElement: 

188 return self._elements[name] 

189 

190 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

191 """Return the `DimensionElement` with the given name or a default. 

192 

193 Parameters 

194 ---------- 

195 name : `str` 

196 Name of the element. 

197 default : `DimensionElement`, optional 

198 Element to return if the named one does not exist. Defaults to 

199 `None`. 

200 

201 Returns 

202 ------- 

203 element : `DimensionElement` 

204 The named element. 

205 """ 

206 return self._elements.get(name, default) 

207 

208 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

209 """Return a set of all static elements in this universe. 

210 

211 Non-static elements that are created as needed may also exist, but 

212 these are guaranteed to have no direct relationships to other elements 

213 (though they may have spatial or temporal relationships). 

214 

215 Returns 

216 ------- 

217 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

218 A frozen set of `DimensionElement` instances. 

219 """ 

220 return self._elements 

221 

222 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

223 """Return a set of all static dimensions in this universe. 

224 

225 Non-static dimensions that are created as needed may also exist, but 

226 these are guaranteed to have no direct relationships to other elements 

227 (though they may have spatial or temporal relationships). 

228 

229 Returns 

230 ------- 

231 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

232 A frozen set of `Dimension` instances. 

233 """ 

234 return self._dimensions 

235 

236 @cached_getter 

237 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

238 """Return a set of all `GovernorDimension` instances in this universe. 

239 

240 Returns 

241 ------- 

242 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

243 A frozen set of `GovernorDimension` instances. 

244 """ 

245 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

246 

247 @cached_getter 

248 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

249 """Return set of all `DatabaseDimensionElement` instances in universe. 

250 

251 This does not include `GovernorDimension` instances, which are backed 

252 by the database but do not inherit from `DatabaseDimensionElement`. 

253 

254 Returns 

255 ------- 

256 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

257 A frozen set of `DatabaseDimensionElement` instances. 

258 """ 

259 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

260 

261 @property # type: ignore 

262 @cached_getter 

263 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

264 """All skypix systems known to this universe. 

265 

266 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

267 """ 

268 return NamedValueSet( 

269 [ 

270 family 

271 for family in self._topology[TopologicalSpace.SPATIAL] 

272 if isinstance(family, SkyPixSystem) 

273 ] 

274 ).freeze() 

275 

276 def getElementIndex(self, name: str) -> int: 

277 """Return the position of the named dimension element. 

278 

279 The position is in this universe's sorting of all elements. 

280 

281 Parameters 

282 ---------- 

283 name : `str` 

284 Name of the element. 

285 

286 Returns 

287 ------- 

288 index : `int` 

289 Sorting index for this element. 

290 """ 

291 return self._elementIndices[name] 

292 

293 def getDimensionIndex(self, name: str) -> int: 

294 """Return the position of the named dimension. 

295 

296 This position is in this universe's sorting of all dimensions. 

297 

298 Parameters 

299 ---------- 

300 name : `str` 

301 Name of the dimension. 

302 

303 Returns 

304 ------- 

305 index : `int` 

306 Sorting index for this dimension. 

307 

308 Notes 

309 ----- 

310 The dimension sort order for a universe is consistent with the element 

311 order (all dimensions are elements), and either can be used to sort 

312 dimensions if used consistently. But there are also some contexts in 

313 which contiguous dimension-only indices are necessary or at least 

314 desirable. 

315 """ 

316 return self._dimensionIndices[name] 

317 

318 def expandDimensionNameSet(self, names: Set[str]) -> None: 

319 """Expand a set of dimension names in-place. 

320 

321 Includes recursive dependencies. 

322 

323 This is an advanced interface for cases where constructing a 

324 `DimensionGraph` (which also expands required dependencies) is 

325 impossible or undesirable. 

326 

327 Parameters 

328 ---------- 

329 names : `set` [ `str` ] 

330 A true `set` of dimension names, to be expanded in-place. 

331 """ 

332 # Keep iterating until the set of names stops growing. This is not as 

333 # efficient as it could be, but we work pretty hard cache 

334 # DimensionGraph instances to keep actual construction rare, so that 

335 # shouldn't matter. 

336 oldSize = len(names) 

337 while True: 

338 # iterate over a temporary copy so we can modify the original 

339 for name in tuple(names): 

340 names.update(self._dimensions[name].required.names) 

341 names.update(self._dimensions[name].implied.names) 

342 if oldSize == len(names): 

343 break 

344 else: 

345 oldSize = len(names) 

346 

347 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

348 """Construct graph from iterable. 

349 

350 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

351 of `Dimension` instances and string names thereof. 

352 

353 Constructing `DimensionGraph` directly from names or dimension 

354 instances is slightly more efficient when it is known in advance that 

355 the iterable is not heterogenous. 

356 

357 Parameters 

358 ---------- 

359 iterable: iterable of `Dimension` or `str` 

360 Dimensions that must be included in the returned graph (their 

361 dependencies will be as well). 

362 

363 Returns 

364 ------- 

365 graph : `DimensionGraph` 

366 A `DimensionGraph` instance containing all given dimensions. 

367 """ 

368 names = set() 

369 for item in iterable: 

370 try: 

371 names.add(item.name) # type: ignore 

372 except AttributeError: 

373 names.add(item) 

374 return DimensionGraph(universe=self, names=names) 

375 

376 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

377 """Return a sorted version of the given iterable of dimension elements. 

378 

379 The universe's sort order is topological (an element's dependencies 

380 precede it), with an unspecified (but deterministic) approach to 

381 breaking ties. 

382 

383 Parameters 

384 ---------- 

385 elements : iterable of `DimensionElement`. 

386 Elements to be sorted. 

387 reverse : `bool`, optional 

388 If `True`, sort in the opposite order. 

389 

390 Returns 

391 ------- 

392 sorted : `list` of `DimensionElement` 

393 A sorted list containing the same elements that were given. 

394 """ 

395 s = set(elements) 

396 result = [element for element in self._elements if element in s or element.name in s] 

397 if reverse: 

398 result.reverse() 

399 # mypy thinks this can return DimensionElements even if all the user 

400 # passed it was Dimensions; we know better. 

401 return result # type: ignore 

402 

403 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

404 """Make a dimension packer. 

405 

406 Constructs a `DimensionPacker` that can pack data ID dictionaries 

407 into unique integers. 

408 

409 Parameters 

410 ---------- 

411 name : `str` 

412 Name of the packer, matching a key in the "packers" section of the 

413 dimension configuration. 

414 dataId : `DataCoordinate` 

415 Fully-expanded data ID that identifies the at least the "fixed" 

416 dimensions of the packer (i.e. those that are assumed/given, 

417 setting the space over which packed integer IDs are unique). 

418 ``dataId.hasRecords()`` must return `True`. 

419 """ 

420 return self._packers[name](self, dataId) 

421 

422 def getEncodeLength(self) -> int: 

423 """Return encoded size of graph. 

424 

425 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

426 instances in this universe. 

427 

428 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

429 information. 

430 """ 

431 return math.ceil(len(self._dimensions) / 8) 

432 

433 @classmethod 

434 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse: 

435 """Return an unpickled dimension universe. 

436 

437 Callable used for unpickling. 

438 

439 For internal use only. 

440 """ 

441 if namespace is None: 

442 # Old pickled universe. 

443 namespace = _DEFAULT_NAMESPACE 

444 try: 

445 return cls._instances[version, namespace] 

446 except KeyError as err: 

447 raise pickle.UnpicklingError( 

448 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

449 f"not found. Note that DimensionUniverse objects are not " 

450 f"truly serialized; when using pickle to transfer them " 

451 f"between processes, an equivalent instance with the same " 

452 f"version must already exist in the receiving process." 

453 ) from err 

454 

455 def __reduce__(self) -> tuple: 

456 return (self._unpickle, (self._version, self._namespace)) 

457 

458 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

459 # DimensionUniverse is recursively immutable; see note in @immutable 

460 # decorator. 

461 return self 

462 

463 # Class attributes below are shadowed by instance attributes, and are 

464 # present just to hold the docstrings for those instance attributes. 

465 

466 empty: DimensionGraph 

467 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

468 """ 

469 

470 commonSkyPix: SkyPixDimension 

471 """The special skypix dimension that is used to relate all other spatial 

472 dimensions in the `Registry` database (`SkyPixDimension`). 

473 """ 

474 

475 dimensionConfig: DimensionConfig 

476 """The configuration used to create this Universe (`DimensionConfig`).""" 

477 

478 _cache: Dict[FrozenSet[str], DimensionGraph] 

479 

480 _dimensions: NamedValueAbstractSet[Dimension] 

481 

482 _elements: NamedValueAbstractSet[DimensionElement] 

483 

484 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

485 

486 _dimensionIndices: Dict[str, int] 

487 

488 _elementIndices: Dict[str, int] 

489 

490 _packers: Dict[str, DimensionPackerFactory] 

491 

492 _version: int 

493 

494 _namespace: Optional[str]