Coverage for python/lsst/daf/butler/dimensions/_universe.py: 46%

195 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-11 03:16 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ["DimensionUniverse"] 

31 

32import logging 

33import math 

34import pickle 

35from collections import defaultdict 

36from collections.abc import Iterable, Mapping, Sequence 

37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, overload 

38 

39from deprecated.sphinx import deprecated 

40from lsst.utils.classes import cached_getter, immutable 

41 

42from .._config import Config 

43from .._named import NamedValueAbstractSet, NamedValueSet 

44from .._topology import TopologicalFamily, TopologicalSpace 

45from .._utilities.thread_safe_cache import ThreadSafeCache 

46from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

47from ._database import DatabaseDimensionElement 

48from ._elements import Dimension, DimensionElement 

49from ._governor import GovernorDimension 

50from ._graph import DimensionGraph 

51from ._group import DimensionGroup 

52from ._skypix import SkyPixDimension, SkyPixSystem 

53 

54if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

55 from .construction import DimensionConstructionBuilder 

56 

57 

58E = TypeVar("E", bound=DimensionElement) 

59_LOG = logging.getLogger(__name__) 

60 

61 

62@immutable 

63class DimensionUniverse: # numpydoc ignore=PR02 

64 """Self-consistent set of dimensions. 

65 

66 A parent class that represents a complete, self-consistent set of 

67 dimensions and their relationships. 

68 

69 `DimensionUniverse` is not a class-level singleton, but all instances are 

70 tracked in a singleton map keyed by the version number and namespace 

71 in the configuration they were loaded from. Because these universes 

72 are solely responsible for constructing `DimensionElement` instances, 

73 these are also indirectly tracked by that singleton as well. 

74 

75 Parameters 

76 ---------- 

77 config : `Config`, optional 

78 Configuration object from which dimension definitions can be extracted. 

79 Ignored if ``builder`` is provided, or if ``version`` is provided and 

80 an instance with that version already exists. 

81 version : `int`, optional 

82 Integer version for this `DimensionUniverse`. If not provided, a 

83 version will be obtained from ``builder`` or ``config``. 

84 namespace : `str`, optional 

85 Namespace of this `DimensionUniverse`, combined with the version 

86 to provide universe safety for registries that use different 

87 dimension definitions. 

88 builder : `DimensionConstructionBuilder`, optional 

89 Builder object used to initialize a new instance. Ignored if 

90 ``version`` is provided and an instance with that version already 

91 exists. Should not have had `~DimensionConstructionBuilder.finish` 

92 called; this will be called if needed by `DimensionUniverse`. 

93 """ 

94 

95 _instances: ClassVar[ThreadSafeCache[tuple[int, str], DimensionUniverse]] = ThreadSafeCache() 

96 """Singleton dictionary of all instances, keyed by version. 

97 

98 For internal use only. 

99 """ 

100 

101 def __new__( 

102 cls, 

103 config: Config | None = None, 

104 *, 

105 version: int | None = None, 

106 namespace: str | None = None, 

107 builder: DimensionConstructionBuilder | None = None, 

108 ) -> DimensionUniverse: 

109 # Try to get a version first, to look for existing instances; try to 

110 # do as little work as possible at this stage. 

111 if version is None: 

112 if builder is None: 

113 config = DimensionConfig(config) 

114 version = config["version"] 

115 else: 

116 version = builder.version 

117 

118 # Then a namespace. 

119 if namespace is None: 

120 if builder is None: 

121 config = DimensionConfig(config) 

122 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

123 else: 

124 namespace = builder.namespace 

125 # if still None use the default 

126 if namespace is None: 

127 namespace = _DEFAULT_NAMESPACE 

128 

129 # See if an equivalent instance already exists. 

130 self: DimensionUniverse | None = cls._instances.get((version, namespace)) 

131 if self is not None: 

132 return self 

133 

134 # Ensure we have a builder, building one from config if necessary. 

135 if builder is None: 

136 config = DimensionConfig(config) 

137 builder = config.makeBuilder() 

138 

139 # Delegate to the builder for most of the construction work. 

140 builder.finish() 

141 

142 # Create the universe instance and create core attributes, mostly 

143 # copying from builder. 

144 self = object.__new__(cls) 

145 assert self is not None 

146 self._cached_groups = ThreadSafeCache() 

147 self._dimensions = builder.dimensions 

148 self._elements = builder.elements 

149 self._topology = builder.topology 

150 self.dimensionConfig = builder.config 

151 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

152 assert isinstance(commonSkyPix, SkyPixDimension) 

153 self.commonSkyPix = commonSkyPix 

154 

155 # Attach self to all elements. 

156 for element in self._elements: 

157 element.universe = self 

158 

159 # Add attribute for special subsets of the graph. 

160 self._empty = DimensionGroup(self, (), _conform=False) 

161 

162 # Use the version number and namespace from the config as a key in 

163 # the singleton dict containing all instances; that will let us 

164 # transfer dimension objects between processes using pickle without 

165 # actually going through real initialization, as long as a universe 

166 # with the same version and namespace has already been constructed in 

167 # the receiving process. 

168 self._version = version 

169 self._namespace = namespace 

170 

171 # Build mappings from element to index. These are used for 

172 # topological-sort comparison operators in DimensionElement itself. 

173 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

174 # Same for dimension to index, sorted topologically across required 

175 # and implied. This is used for encode/decode. 

176 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

177 

178 self._populates = defaultdict(NamedValueSet) 

179 for element in self._elements: 

180 if element.populated_by is not None: 

181 self._populates[element.populated_by.name].add(element) 

182 

183 return cls._instances.set_or_get((self._version, self._namespace), self) 

184 

185 @property 

186 def version(self) -> int: 

187 """The version number of this universe. 

188 

189 Returns 

190 ------- 

191 version : `int` 

192 An integer representing the version number of this universe. 

193 Uniquely defined when combined with the `namespace`. 

194 """ 

195 return self._version 

196 

197 @property 

198 def namespace(self) -> str: 

199 """The namespace associated with this universe. 

200 

201 Returns 

202 ------- 

203 namespace : `str` 

204 The namespace. When combined with the `version` can uniquely 

205 define this universe. 

206 """ 

207 return self._namespace 

208 

209 def isCompatibleWith(self, other: DimensionUniverse) -> bool: 

210 """Check compatibility between this `DimensionUniverse` and another. 

211 

212 Parameters 

213 ---------- 

214 other : `DimensionUniverse` 

215 The other `DimensionUniverse` to check for compatibility. 

216 

217 Returns 

218 ------- 

219 results : `bool` 

220 If the other `DimensionUniverse` is compatible with this one return 

221 `True`, else `False`. 

222 """ 

223 # Different namespaces mean that these universes cannot be compatible. 

224 if self.namespace != other.namespace: 

225 return False 

226 if self.version != other.version: 

227 _LOG.info( 

228 "Universes share a namespace %r but have differing versions (%d != %d). " 

229 " This could be okay but may be responsible for dimension errors later.", 

230 self.namespace, 

231 self.version, 

232 other.version, 

233 ) 

234 

235 # For now assume compatibility if versions differ. 

236 return True 

237 

238 def __repr__(self) -> str: 

239 return f"DimensionUniverse({self._version}, {self._namespace})" 

240 

241 def __getitem__(self, name: str) -> DimensionElement: 

242 return self._elements[name] 

243 

244 def __contains__(self, name: Any) -> bool: 

245 return name in self._elements 

246 

247 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None: 

248 """Return the `DimensionElement` with the given name or a default. 

249 

250 Parameters 

251 ---------- 

252 name : `str` 

253 Name of the element. 

254 default : `DimensionElement`, optional 

255 Element to return if the named one does not exist. Defaults to 

256 `None`. 

257 

258 Returns 

259 ------- 

260 element : `DimensionElement` 

261 The named element. 

262 """ 

263 return self._elements.get(name, default) 

264 

265 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

266 """Return a set of all static elements in this universe. 

267 

268 Non-static elements that are created as needed may also exist, but 

269 these are guaranteed to have no direct relationships to other elements 

270 (though they may have spatial or temporal relationships). 

271 

272 Returns 

273 ------- 

274 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

275 A frozen set of `DimensionElement` instances. 

276 """ 

277 return self._elements 

278 

279 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

280 """Return a set of all static dimensions in this universe. 

281 

282 Non-static dimensions that are created as needed may also exist, but 

283 these are guaranteed to have no direct relationships to other elements 

284 (though they may have spatial or temporal relationships). 

285 

286 Returns 

287 ------- 

288 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

289 A frozen set of `Dimension` instances. 

290 """ 

291 return self._dimensions 

292 

293 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

294 """Return a set of all `GovernorDimension` instances in this universe. 

295 

296 Returns 

297 ------- 

298 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

299 A frozen set of `GovernorDimension` instances. 

300 """ 

301 return self.governor_dimensions 

302 

303 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

304 """Return set of all `DatabaseDimensionElement` instances in universe. 

305 

306 This does not include `GovernorDimension` instances, which are backed 

307 by the database but do not inherit from `DatabaseDimensionElement`. 

308 

309 Returns 

310 ------- 

311 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

312 A frozen set of `DatabaseDimensionElement` instances. 

313 """ 

314 return self.database_elements 

315 

316 @property 

317 def elements(self) -> NamedValueAbstractSet[DimensionElement]: 

318 """All dimension elements defined in this universe.""" 

319 return self._elements 

320 

321 @property 

322 def dimensions(self) -> NamedValueAbstractSet[Dimension]: 

323 """All dimensions defined in this universe.""" 

324 return self._dimensions 

325 

326 @property 

327 @cached_getter 

328 def governor_dimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

329 """All governor dimensions defined in this universe. 

330 

331 Governor dimensions serve as special required dependencies of other 

332 dimensions, with special handling in dimension query expressions and 

333 collection summaries. Governor dimension records are stored in the 

334 database but the set of such values is expected to be small enough 

335 for all values to be cached by all clients. 

336 """ 

337 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

338 

339 @property 

340 @cached_getter 

341 def skypix_dimensions(self) -> NamedValueAbstractSet[SkyPixDimension]: 

342 """All skypix dimensions defined in this universe. 

343 

344 Skypix dimension records are always generated on-the-fly rather than 

345 stored in the database, and they always represent a tiling of the sky 

346 with no overlaps. 

347 """ 

348 result = NamedValueSet[SkyPixDimension]() 

349 for system in self.skypix: 

350 result.update(system) 

351 return result.freeze() 

352 

353 @property 

354 @cached_getter 

355 def database_elements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

356 """All dimension elements whose records are stored in the database, 

357 except governor dimensions. 

358 """ 

359 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

360 

361 @property 

362 @cached_getter 

363 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

364 """All skypix systems known to this universe. 

365 

366 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

367 """ 

368 return NamedValueSet( 

369 [ 

370 family 

371 for family in self._topology[TopologicalSpace.SPATIAL] 

372 if isinstance(family, SkyPixSystem) 

373 ] 

374 ).freeze() 

375 

376 def getElementIndex(self, name: str) -> int: 

377 """Return the position of the named dimension element. 

378 

379 The position is in this universe's sorting of all elements. 

380 

381 Parameters 

382 ---------- 

383 name : `str` 

384 Name of the element. 

385 

386 Returns 

387 ------- 

388 index : `int` 

389 Sorting index for this element. 

390 """ 

391 return self._elementIndices[name] 

392 

393 def getDimensionIndex(self, name: str) -> int: 

394 """Return the position of the named dimension. 

395 

396 This position is in this universe's sorting of all dimensions. 

397 

398 Parameters 

399 ---------- 

400 name : `str` 

401 Name of the dimension. 

402 

403 Returns 

404 ------- 

405 index : `int` 

406 Sorting index for this dimension. 

407 

408 Notes 

409 ----- 

410 The dimension sort order for a universe is consistent with the element 

411 order (all dimensions are elements), and either can be used to sort 

412 dimensions if used consistently. But there are also some contexts in 

413 which contiguous dimension-only indices are necessary or at least 

414 desirable. 

415 """ 

416 return self._dimensionIndices[name] 

417 

418 # TODO: remove on DM-41326. 

419 @deprecated( 

420 "Deprecated in favor of DimensionUniverse.conform, and will be removed after v27.", 

421 version="v27", 

422 category=FutureWarning, 

423 ) 

424 def expandDimensionNameSet(self, names: set[str]) -> None: 

425 """Expand a set of dimension names in-place. 

426 

427 Includes recursive dependencies. 

428 

429 This is an advanced interface for cases where constructing a 

430 `DimensionGraph` (which also expands required dependencies) is 

431 impossible or undesirable. 

432 

433 Parameters 

434 ---------- 

435 names : `set` [ `str` ] 

436 A true `set` of dimension names, to be expanded in-place. 

437 """ 

438 # Keep iterating until the set of names stops growing. This is not as 

439 # efficient as it could be, but we work pretty hard cache 

440 # DimensionGraph instances to keep actual construction rare, so that 

441 # shouldn't matter. 

442 oldSize = len(names) 

443 while True: 

444 # iterate over a temporary copy so we can modify the original 

445 for name in tuple(names): 

446 names.update(self._dimensions[name].required.names) 

447 names.update(self._dimensions[name].implied.names) 

448 if oldSize == len(names): 

449 break 

450 else: 

451 oldSize = len(names) 

452 

453 # TODO: remove on DM-41326. 

454 @deprecated( 

455 "DimensionUniverse.extract and DimensionGraph are deprecated in favor of DimensionUniverse.conform " 

456 "and DimensionGroup, and will be removed after v27.", 

457 version="v27", 

458 category=FutureWarning, 

459 ) 

460 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph: 

461 """Construct graph from iterable. 

462 

463 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

464 of `Dimension` instances and string names thereof. 

465 

466 Constructing `DimensionGraph` directly from names or dimension 

467 instances is slightly more efficient when it is known in advance that 

468 the iterable is not heterogenous. 

469 

470 Parameters 

471 ---------- 

472 iterable : iterable of `Dimension` or `str` 

473 Dimensions that must be included in the returned graph (their 

474 dependencies will be as well). 

475 

476 Returns 

477 ------- 

478 graph : `DimensionGraph` 

479 A `DimensionGraph` instance containing all given dimensions. 

480 """ 

481 return self.conform(iterable)._as_graph() 

482 

483 def conform( 

484 self, 

485 dimensions: Iterable[str | Dimension] | str | DimensionElement | DimensionGroup | DimensionGraph, 

486 /, 

487 ) -> DimensionGroup: 

488 """Construct a dimension group from an iterable of dimension names. 

489 

490 Parameters 

491 ---------- 

492 dimensions : `~collections.abc.Iterable` [ `str` or `Dimension` ], \ 

493 `str`, `DimensionElement`, `DimensionGroup`, or \ 

494 `DimensionGraph` 

495 Dimensions that must be included in the returned group; their 

496 dependencies will be as well. Support for `Dimension`, 

497 `DimensionElement` and `DimensionGraph` objects is deprecated and 

498 will be removed after v27. Passing `DimensionGraph` objects will 

499 not yield a deprecation warning to allow non-deprecated methods and 

500 properties that return `DimensionGraph` objects to be passed 

501 though, since these will be changed to return `DimensionGroup` in 

502 the future. 

503 

504 Returns 

505 ------- 

506 group : `DimensionGroup` 

507 A `DimensionGroup` instance containing all given dimensions. 

508 """ 

509 match dimensions: 

510 case DimensionGroup(): 

511 return dimensions 

512 case DimensionGraph(): 

513 return dimensions.as_group() 

514 case DimensionElement() as d: 

515 return d.minimal_group 

516 case str() as name: 

517 return self[name].minimal_group 

518 case iterable: 

519 names: set[str] = {getattr(d, "name", cast(str, d)) for d in iterable} 

520 return DimensionGroup(self, names) 

521 

522 @overload 

523 def sorted(self, elements: Iterable[Dimension], *, reverse: bool = False) -> Sequence[Dimension]: ... 523 ↛ exitline 523 didn't return from function 'sorted', because

524 

525 @overload 

526 def sorted( 526 ↛ exitline 526 didn't jump to the function exit

527 self, elements: Iterable[DimensionElement | str], *, reverse: bool = False 

528 ) -> Sequence[DimensionElement]: ... 

529 

530 def sorted(self, elements: Iterable[Any], *, reverse: bool = False) -> list[Any]: 

531 """Return a sorted version of the given iterable of dimension elements. 

532 

533 The universe's sort order is topological (an element's dependencies 

534 precede it), with an unspecified (but deterministic) approach to 

535 breaking ties. 

536 

537 Parameters 

538 ---------- 

539 elements : iterable of `DimensionElement` 

540 Elements to be sorted. 

541 reverse : `bool`, optional 

542 If `True`, sort in the opposite order. 

543 

544 Returns 

545 ------- 

546 sorted : `~collections.abc.Sequence` [ `Dimension` or \ 

547 `DimensionElement` ] 

548 A sorted sequence containing the same elements that were given. 

549 """ 

550 s = set(elements) 

551 result = [element for element in self._elements if element in s or element.name in s] 

552 if reverse: 

553 result.reverse() 

554 return result 

555 

556 def getEncodeLength(self) -> int: 

557 """Return encoded size of graph. 

558 

559 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

560 instances in this universe. 

561 

562 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

563 information. 

564 """ 

565 return math.ceil(len(self._dimensions) / 8) 

566 

567 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]: 

568 """Return the set of `DimensionElement` objects whose 

569 `~DimensionElement.populated_by` attribute is the given dimension. 

570 

571 Parameters 

572 ---------- 

573 dimension : `Dimension` 

574 The dimension of interest. 

575 

576 Returns 

577 ------- 

578 populated_by : `NamedValueAbstractSet` [ `DimensionElement` ] 

579 The set of elements who say they are populated by the given 

580 dimension. 

581 """ 

582 return self._populates[dimension.name] 

583 

584 @property 

585 def empty(self) -> DimensionGraph: 

586 """The `DimensionGraph` that contains no dimensions. 

587 

588 After v27 this will be a `DimensionGroup`. 

589 """ 

590 return self._empty._as_graph() 

591 

592 @classmethod 

593 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse: 

594 """Return an unpickled dimension universe. 

595 

596 Callable used for unpickling. 

597 

598 For internal use only. 

599 """ 

600 if namespace is None: 

601 # Old pickled universe. 

602 namespace = _DEFAULT_NAMESPACE 

603 instance = cls._instances.get((version, namespace)) 

604 if instance is None: 

605 raise pickle.UnpicklingError( 

606 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

607 "not found. Note that DimensionUniverse objects are not " 

608 "truly serialized; when using pickle to transfer them " 

609 "between processes, an equivalent instance with the same " 

610 "version must already exist in the receiving process." 

611 ) 

612 return instance 

613 

614 def __reduce__(self) -> tuple: 

615 return (self._unpickle, (self._version, self._namespace)) 

616 

617 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

618 # DimensionUniverse is recursively immutable; see note in @immutable 

619 # decorator. 

620 return self 

621 

622 # Class attributes below are shadowed by instance attributes, and are 

623 # present just to hold the docstrings for those instance attributes. 

624 

625 commonSkyPix: SkyPixDimension 

626 """The special skypix dimension that is used to relate all other spatial 

627 dimensions in the `Registry` database (`SkyPixDimension`). 

628 """ 

629 

630 dimensionConfig: DimensionConfig 

631 """The configuration used to create this Universe (`DimensionConfig`).""" 

632 

633 _cached_groups: ThreadSafeCache[frozenset[str], DimensionGroup] 

634 

635 _dimensions: NamedValueAbstractSet[Dimension] 

636 

637 _elements: NamedValueAbstractSet[DimensionElement] 

638 

639 _empty: DimensionGroup 

640 

641 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

642 

643 _dimensionIndices: dict[str, int] 

644 

645 _elementIndices: dict[str, int] 

646 

647 _populates: defaultdict[str, NamedValueSet[DimensionElement]] 

648 

649 _version: int 

650 

651 _namespace: str