Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 41%

157 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 09:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import logging 

27import math 

28import pickle 

29from collections.abc import Iterable, Mapping 

30from typing import TYPE_CHECKING, Any, ClassVar, TypeVar 

31 

32from deprecated.sphinx import deprecated 

33from lsst.utils.classes import cached_getter, immutable 

34 

35from .._topology import TopologicalFamily, TopologicalSpace 

36from ..config import Config 

37from ..named import NamedValueAbstractSet, NamedValueSet 

38from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

39from ._database import DatabaseDimensionElement 

40from ._elements import Dimension, DimensionElement 

41from ._governor import GovernorDimension 

42from ._graph import DimensionGraph 

43from ._skypix import SkyPixDimension, SkyPixSystem 

44 

45if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

46 from ._coordinate import DataCoordinate 

47 from ._packer import DimensionPacker, DimensionPackerFactory 

48 from .construction import DimensionConstructionBuilder 

49 

50 

51E = TypeVar("E", bound=DimensionElement) 

52_LOG = logging.getLogger(__name__) 

53 

54 

55@immutable 

56class DimensionUniverse: 

57 """Self-consistent set of dimensions. 

58 

59 A parent class that represents a complete, self-consistent set of 

60 dimensions and their relationships. 

61 

62 `DimensionUniverse` is not a class-level singleton, but all instances are 

63 tracked in a singleton map keyed by the version number and namespace 

64 in the configuration they were loaded from. Because these universes 

65 are solely responsible for constructing `DimensionElement` instances, 

66 these are also indirectly tracked by that singleton as well. 

67 

68 Parameters 

69 ---------- 

70 config : `Config`, optional 

71 Configuration object from which dimension definitions can be extracted. 

72 Ignored if ``builder`` is provided, or if ``version`` is provided and 

73 an instance with that version already exists. 

74 version : `int`, optional 

75 Integer version for this `DimensionUniverse`. If not provided, a 

76 version will be obtained from ``builder`` or ``config``. 

77 namespace : `str`, optional 

78 Namespace of this `DimensionUniverse`, combined with the version 

79 to provide universe safety for registries that use different 

80 dimension definitions. 

81 builder : `DimensionConstructionBuilder`, optional 

82 Builder object used to initialize a new instance. Ignored if 

83 ``version`` is provided and an instance with that version already 

84 exists. Should not have had `~DimensionConstructionBuilder.finish` 

85 called; this will be called if needed by `DimensionUniverse`. 

86 """ 

87 

88 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {} 

89 """Singleton dictionary of all instances, keyed by version. 

90 

91 For internal use only. 

92 """ 

93 

94 def __new__( 

95 cls, 

96 config: Config | None = None, 

97 *, 

98 version: int | None = None, 

99 namespace: str | None = None, 

100 builder: DimensionConstructionBuilder | None = None, 

101 ) -> DimensionUniverse: 

102 # Try to get a version first, to look for existing instances; try to 

103 # do as little work as possible at this stage. 

104 if version is None: 

105 if builder is None: 

106 config = DimensionConfig(config) 

107 version = config["version"] 

108 else: 

109 version = builder.version 

110 

111 # Then a namespace. 

112 if namespace is None: 

113 if builder is None: 

114 config = DimensionConfig(config) 

115 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

116 else: 

117 namespace = builder.namespace 

118 # if still None use the default 

119 if namespace is None: 

120 namespace = _DEFAULT_NAMESPACE 

121 

122 # See if an equivalent instance already exists. 

123 self: DimensionUniverse | None = cls._instances.get((version, namespace)) 

124 if self is not None: 

125 return self 

126 

127 # Ensure we have a builder, building one from config if necessary. 

128 if builder is None: 

129 config = DimensionConfig(config) 

130 builder = config.makeBuilder() 

131 

132 # Delegate to the builder for most of the construction work. 

133 builder.finish() 

134 

135 # Create the universe instance and create core attributes, mostly 

136 # copying from builder. 

137 self = object.__new__(cls) 

138 assert self is not None 

139 self._cache = {} 

140 self._dimensions = builder.dimensions 

141 self._elements = builder.elements 

142 self._topology = builder.topology 

143 self._packers = builder.packers 

144 self.dimensionConfig = builder.config 

145 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

146 assert isinstance(commonSkyPix, SkyPixDimension) 

147 self.commonSkyPix = commonSkyPix 

148 

149 # Attach self to all elements. 

150 for element in self._elements: 

151 element.universe = self 

152 

153 # Add attribute for special subsets of the graph. 

154 self.empty = DimensionGraph(self, (), conform=False) 

155 

156 # Use the version number and namespace from the config as a key in 

157 # the singleton dict containing all instances; that will let us 

158 # transfer dimension objects between processes using pickle without 

159 # actually going through real initialization, as long as a universe 

160 # with the same version and namespace has already been constructed in 

161 # the receiving process. 

162 self._version = version 

163 self._namespace = namespace 

164 cls._instances[self._version, self._namespace] = self 

165 

166 # Build mappings from element to index. These are used for 

167 # topological-sort comparison operators in DimensionElement itself. 

168 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

169 # Same for dimension to index, sorted topologically across required 

170 # and implied. This is used for encode/decode. 

171 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

172 

173 return self 

174 

175 @property 

176 def version(self) -> int: 

177 """The version number of this universe. 

178 

179 Returns 

180 ------- 

181 version : `int` 

182 An integer representing the version number of this universe. 

183 Uniquely defined when combined with the `namespace`. 

184 """ 

185 return self._version 

186 

187 @property 

188 def namespace(self) -> str: 

189 """The namespace associated with this universe. 

190 

191 Returns 

192 ------- 

193 namespace : `str` 

194 The namespace. When combined with the `version` can uniquely 

195 define this universe. 

196 """ 

197 return self._namespace 

198 

199 def isCompatibleWith(self, other: DimensionUniverse) -> bool: 

200 """Check compatibility between this `DimensionUniverse` and another. 

201 

202 Parameters 

203 ---------- 

204 other : `DimensionUniverse` 

205 The other `DimensionUniverse` to check for compatibility 

206 

207 Returns 

208 ------- 

209 results : `bool` 

210 If the other `DimensionUniverse` is compatible with this one return 

211 `True`, else `False` 

212 """ 

213 # Different namespaces mean that these universes cannot be compatible. 

214 if self.namespace != other.namespace: 

215 return False 

216 if self.version != other.version: 

217 _LOG.info( 

218 "Universes share a namespace %r but have differing versions (%d != %d). " 

219 " This could be okay but may be responsible for dimension errors later.", 

220 self.namespace, 

221 self.version, 

222 other.version, 

223 ) 

224 

225 # For now assume compatibility if versions differ. 

226 return True 

227 

228 def __repr__(self) -> str: 

229 return f"DimensionUniverse({self._version}, {self._namespace})" 

230 

231 def __getitem__(self, name: str) -> DimensionElement: 

232 return self._elements[name] 

233 

234 def __contains__(self, name: Any) -> bool: 

235 return name in self._elements 

236 

237 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None: 

238 """Return the `DimensionElement` with the given name or a default. 

239 

240 Parameters 

241 ---------- 

242 name : `str` 

243 Name of the element. 

244 default : `DimensionElement`, optional 

245 Element to return if the named one does not exist. Defaults to 

246 `None`. 

247 

248 Returns 

249 ------- 

250 element : `DimensionElement` 

251 The named element. 

252 """ 

253 return self._elements.get(name, default) 

254 

255 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

256 """Return a set of all static elements in this universe. 

257 

258 Non-static elements that are created as needed may also exist, but 

259 these are guaranteed to have no direct relationships to other elements 

260 (though they may have spatial or temporal relationships). 

261 

262 Returns 

263 ------- 

264 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

265 A frozen set of `DimensionElement` instances. 

266 """ 

267 return self._elements 

268 

269 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

270 """Return a set of all static dimensions in this universe. 

271 

272 Non-static dimensions that are created as needed may also exist, but 

273 these are guaranteed to have no direct relationships to other elements 

274 (though they may have spatial or temporal relationships). 

275 

276 Returns 

277 ------- 

278 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

279 A frozen set of `Dimension` instances. 

280 """ 

281 return self._dimensions 

282 

283 @cached_getter 

284 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

285 """Return a set of all `GovernorDimension` instances in this universe. 

286 

287 Returns 

288 ------- 

289 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

290 A frozen set of `GovernorDimension` instances. 

291 """ 

292 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

293 

294 @cached_getter 

295 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

296 """Return set of all `DatabaseDimensionElement` instances in universe. 

297 

298 This does not include `GovernorDimension` instances, which are backed 

299 by the database but do not inherit from `DatabaseDimensionElement`. 

300 

301 Returns 

302 ------- 

303 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

304 A frozen set of `DatabaseDimensionElement` instances. 

305 """ 

306 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

307 

308 @property 

309 @cached_getter 

310 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

311 """All skypix systems known to this universe. 

312 

313 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

314 """ 

315 return NamedValueSet( 

316 [ 

317 family 

318 for family in self._topology[TopologicalSpace.SPATIAL] 

319 if isinstance(family, SkyPixSystem) 

320 ] 

321 ).freeze() 

322 

323 def getElementIndex(self, name: str) -> int: 

324 """Return the position of the named dimension element. 

325 

326 The position is in this universe's sorting of all elements. 

327 

328 Parameters 

329 ---------- 

330 name : `str` 

331 Name of the element. 

332 

333 Returns 

334 ------- 

335 index : `int` 

336 Sorting index for this element. 

337 """ 

338 return self._elementIndices[name] 

339 

340 def getDimensionIndex(self, name: str) -> int: 

341 """Return the position of the named dimension. 

342 

343 This position is in this universe's sorting of all dimensions. 

344 

345 Parameters 

346 ---------- 

347 name : `str` 

348 Name of the dimension. 

349 

350 Returns 

351 ------- 

352 index : `int` 

353 Sorting index for this dimension. 

354 

355 Notes 

356 ----- 

357 The dimension sort order for a universe is consistent with the element 

358 order (all dimensions are elements), and either can be used to sort 

359 dimensions if used consistently. But there are also some contexts in 

360 which contiguous dimension-only indices are necessary or at least 

361 desirable. 

362 """ 

363 return self._dimensionIndices[name] 

364 

365 def expandDimensionNameSet(self, names: set[str]) -> None: 

366 """Expand a set of dimension names in-place. 

367 

368 Includes recursive dependencies. 

369 

370 This is an advanced interface for cases where constructing a 

371 `DimensionGraph` (which also expands required dependencies) is 

372 impossible or undesirable. 

373 

374 Parameters 

375 ---------- 

376 names : `set` [ `str` ] 

377 A true `set` of dimension names, to be expanded in-place. 

378 """ 

379 # Keep iterating until the set of names stops growing. This is not as 

380 # efficient as it could be, but we work pretty hard cache 

381 # DimensionGraph instances to keep actual construction rare, so that 

382 # shouldn't matter. 

383 oldSize = len(names) 

384 while True: 

385 # iterate over a temporary copy so we can modify the original 

386 for name in tuple(names): 

387 names.update(self._dimensions[name].required.names) 

388 names.update(self._dimensions[name].implied.names) 

389 if oldSize == len(names): 

390 break 

391 else: 

392 oldSize = len(names) 

393 

394 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph: 

395 """Construct graph from iterable. 

396 

397 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

398 of `Dimension` instances and string names thereof. 

399 

400 Constructing `DimensionGraph` directly from names or dimension 

401 instances is slightly more efficient when it is known in advance that 

402 the iterable is not heterogenous. 

403 

404 Parameters 

405 ---------- 

406 iterable: iterable of `Dimension` or `str` 

407 Dimensions that must be included in the returned graph (their 

408 dependencies will be as well). 

409 

410 Returns 

411 ------- 

412 graph : `DimensionGraph` 

413 A `DimensionGraph` instance containing all given dimensions. 

414 """ 

415 names = set() 

416 for item in iterable: 

417 try: 

418 names.add(item.name) # type: ignore 

419 except AttributeError: 

420 names.add(item) 

421 return DimensionGraph(universe=self, names=names) 

422 

423 def sorted(self, elements: Iterable[E | str], *, reverse: bool = False) -> list[E]: 

424 """Return a sorted version of the given iterable of dimension elements. 

425 

426 The universe's sort order is topological (an element's dependencies 

427 precede it), with an unspecified (but deterministic) approach to 

428 breaking ties. 

429 

430 Parameters 

431 ---------- 

432 elements : iterable of `DimensionElement`. 

433 Elements to be sorted. 

434 reverse : `bool`, optional 

435 If `True`, sort in the opposite order. 

436 

437 Returns 

438 ------- 

439 sorted : `list` of `DimensionElement` 

440 A sorted list containing the same elements that were given. 

441 """ 

442 s = set(elements) 

443 result = [element for element in self._elements if element in s or element.name in s] 

444 if reverse: 

445 result.reverse() 

446 # mypy thinks this can return DimensionElements even if all the user 

447 # passed it was Dimensions; we know better. 

448 return result # type: ignore 

449 

450 # TODO: Remove this method on DM-38687. 

451 @deprecated( 

452 "Deprecated in favor of configurable dimension packers. Will be removed after v27.", 

453 version="v26", 

454 category=FutureWarning, 

455 ) 

456 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

457 """Make a dimension packer. 

458 

459 Constructs a `DimensionPacker` that can pack data ID dictionaries 

460 into unique integers. 

461 

462 Parameters 

463 ---------- 

464 name : `str` 

465 Name of the packer, matching a key in the "packers" section of the 

466 dimension configuration. 

467 dataId : `DataCoordinate` 

468 Fully-expanded data ID that identifies the at least the "fixed" 

469 dimensions of the packer (i.e. those that are assumed/given, 

470 setting the space over which packed integer IDs are unique). 

471 ``dataId.hasRecords()`` must return `True`. 

472 """ 

473 return self._packers[name](self, dataId) 

474 

475 def getEncodeLength(self) -> int: 

476 """Return encoded size of graph. 

477 

478 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

479 instances in this universe. 

480 

481 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

482 information. 

483 """ 

484 return math.ceil(len(self._dimensions) / 8) 

485 

486 @classmethod 

487 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse: 

488 """Return an unpickled dimension universe. 

489 

490 Callable used for unpickling. 

491 

492 For internal use only. 

493 """ 

494 if namespace is None: 

495 # Old pickled universe. 

496 namespace = _DEFAULT_NAMESPACE 

497 try: 

498 return cls._instances[version, namespace] 

499 except KeyError as err: 

500 raise pickle.UnpicklingError( 

501 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

502 "not found. Note that DimensionUniverse objects are not " 

503 "truly serialized; when using pickle to transfer them " 

504 "between processes, an equivalent instance with the same " 

505 "version must already exist in the receiving process." 

506 ) from err 

507 

508 def __reduce__(self) -> tuple: 

509 return (self._unpickle, (self._version, self._namespace)) 

510 

511 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

512 # DimensionUniverse is recursively immutable; see note in @immutable 

513 # decorator. 

514 return self 

515 

516 # Class attributes below are shadowed by instance attributes, and are 

517 # present just to hold the docstrings for those instance attributes. 

518 

519 empty: DimensionGraph 

520 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

521 """ 

522 

523 commonSkyPix: SkyPixDimension 

524 """The special skypix dimension that is used to relate all other spatial 

525 dimensions in the `Registry` database (`SkyPixDimension`). 

526 """ 

527 

528 dimensionConfig: DimensionConfig 

529 """The configuration used to create this Universe (`DimensionConfig`).""" 

530 

531 _cache: dict[frozenset[str], DimensionGraph] 

532 

533 _dimensions: NamedValueAbstractSet[Dimension] 

534 

535 _elements: NamedValueAbstractSet[DimensionElement] 

536 

537 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

538 

539 _dimensionIndices: dict[str, int] 

540 

541 _elementIndices: dict[str, int] 

542 

543 _packers: dict[str, DimensionPackerFactory] 

544 

545 _version: int 

546 

547 _namespace: str