Coverage for python/lsst/daf/butler/dimensions/_universe.py: 45%

197 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ["DimensionUniverse"] 

31 

32import logging 

33import math 

34import pickle 

35from collections import defaultdict 

36from collections.abc import Iterable, Mapping, Sequence 

37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, overload 

38 

39from deprecated.sphinx import deprecated 

40from lsst.utils.classes import cached_getter, immutable 

41 

42from .._config import Config 

43from .._named import NamedValueAbstractSet, NamedValueSet 

44from .._topology import TopologicalFamily, TopologicalSpace 

45from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

46from ._database import DatabaseDimensionElement 

47from ._elements import Dimension, DimensionElement 

48from ._governor import GovernorDimension 

49from ._graph import DimensionGraph 

50from ._group import DimensionGroup 

51from ._skypix import SkyPixDimension, SkyPixSystem 

52 

53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

54 from .construction import DimensionConstructionBuilder 

55 

56 

57E = TypeVar("E", bound=DimensionElement) 

58_LOG = logging.getLogger(__name__) 

59 

60 

61@immutable 

62class DimensionUniverse: # numpydoc ignore=PR02 

63 """Self-consistent set of dimensions. 

64 

65 A parent class that represents a complete, self-consistent set of 

66 dimensions and their relationships. 

67 

68 `DimensionUniverse` is not a class-level singleton, but all instances are 

69 tracked in a singleton map keyed by the version number and namespace 

70 in the configuration they were loaded from. Because these universes 

71 are solely responsible for constructing `DimensionElement` instances, 

72 these are also indirectly tracked by that singleton as well. 

73 

74 Parameters 

75 ---------- 

76 config : `Config`, optional 

77 Configuration object from which dimension definitions can be extracted. 

78 Ignored if ``builder`` is provided, or if ``version`` is provided and 

79 an instance with that version already exists. 

80 version : `int`, optional 

81 Integer version for this `DimensionUniverse`. If not provided, a 

82 version will be obtained from ``builder`` or ``config``. 

83 namespace : `str`, optional 

84 Namespace of this `DimensionUniverse`, combined with the version 

85 to provide universe safety for registries that use different 

86 dimension definitions. 

87 builder : `DimensionConstructionBuilder`, optional 

88 Builder object used to initialize a new instance. Ignored if 

89 ``version`` is provided and an instance with that version already 

90 exists. Should not have had `~DimensionConstructionBuilder.finish` 

91 called; this will be called if needed by `DimensionUniverse`. 

92 """ 

93 

94 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {} 

95 """Singleton dictionary of all instances, keyed by version. 

96 

97 For internal use only. 

98 """ 

99 

100 def __new__( 

101 cls, 

102 config: Config | None = None, 

103 *, 

104 version: int | None = None, 

105 namespace: str | None = None, 

106 builder: DimensionConstructionBuilder | None = None, 

107 ) -> DimensionUniverse: 

108 # Try to get a version first, to look for existing instances; try to 

109 # do as little work as possible at this stage. 

110 if version is None: 

111 if builder is None: 

112 config = DimensionConfig(config) 

113 version = config["version"] 

114 else: 

115 version = builder.version 

116 

117 # Then a namespace. 

118 if namespace is None: 

119 if builder is None: 

120 config = DimensionConfig(config) 

121 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

122 else: 

123 namespace = builder.namespace 

124 # if still None use the default 

125 if namespace is None: 

126 namespace = _DEFAULT_NAMESPACE 

127 

128 # See if an equivalent instance already exists. 

129 self: DimensionUniverse | None = cls._instances.get((version, namespace)) 

130 if self is not None: 

131 return self 

132 

133 # Ensure we have a builder, building one from config if necessary. 

134 if builder is None: 

135 config = DimensionConfig(config) 

136 builder = config.makeBuilder() 

137 

138 # Delegate to the builder for most of the construction work. 

139 builder.finish() 

140 

141 # Create the universe instance and create core attributes, mostly 

142 # copying from builder. 

143 self = object.__new__(cls) 

144 assert self is not None 

145 self._cached_groups = {} 

146 self._dimensions = builder.dimensions 

147 self._elements = builder.elements 

148 self._topology = builder.topology 

149 self.dimensionConfig = builder.config 

150 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

151 assert isinstance(commonSkyPix, SkyPixDimension) 

152 self.commonSkyPix = commonSkyPix 

153 

154 # Attach self to all elements. 

155 for element in self._elements: 

156 element.universe = self 

157 

158 # Add attribute for special subsets of the graph. 

159 self._empty = DimensionGroup(self, (), _conform=False) 

160 

161 # Use the version number and namespace from the config as a key in 

162 # the singleton dict containing all instances; that will let us 

163 # transfer dimension objects between processes using pickle without 

164 # actually going through real initialization, as long as a universe 

165 # with the same version and namespace has already been constructed in 

166 # the receiving process. 

167 self._version = version 

168 self._namespace = namespace 

169 cls._instances[self._version, self._namespace] = self 

170 

171 # Build mappings from element to index. These are used for 

172 # topological-sort comparison operators in DimensionElement itself. 

173 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

174 # Same for dimension to index, sorted topologically across required 

175 # and implied. This is used for encode/decode. 

176 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

177 

178 self._populates = defaultdict(NamedValueSet) 

179 for element in self._elements: 

180 if element.populated_by is not None: 

181 self._populates[element.populated_by.name].add(element) 

182 

183 return self 

184 

185 @property 

186 def version(self) -> int: 

187 """The version number of this universe. 

188 

189 Returns 

190 ------- 

191 version : `int` 

192 An integer representing the version number of this universe. 

193 Uniquely defined when combined with the `namespace`. 

194 """ 

195 return self._version 

196 

197 @property 

198 def namespace(self) -> str: 

199 """The namespace associated with this universe. 

200 

201 Returns 

202 ------- 

203 namespace : `str` 

204 The namespace. When combined with the `version` can uniquely 

205 define this universe. 

206 """ 

207 return self._namespace 

208 

209 def isCompatibleWith(self, other: DimensionUniverse) -> bool: 

210 """Check compatibility between this `DimensionUniverse` and another. 

211 

212 Parameters 

213 ---------- 

214 other : `DimensionUniverse` 

215 The other `DimensionUniverse` to check for compatibility. 

216 

217 Returns 

218 ------- 

219 results : `bool` 

220 If the other `DimensionUniverse` is compatible with this one return 

221 `True`, else `False`. 

222 """ 

223 # Different namespaces mean that these universes cannot be compatible. 

224 if self.namespace != other.namespace: 

225 return False 

226 if self.version != other.version: 

227 _LOG.info( 

228 "Universes share a namespace %r but have differing versions (%d != %d). " 

229 " This could be okay but may be responsible for dimension errors later.", 

230 self.namespace, 

231 self.version, 

232 other.version, 

233 ) 

234 

235 # For now assume compatibility if versions differ. 

236 return True 

237 

238 def __repr__(self) -> str: 

239 return f"DimensionUniverse({self._version}, {self._namespace})" 

240 

241 def __getitem__(self, name: str) -> DimensionElement: 

242 return self._elements[name] 

243 

244 def __contains__(self, name: Any) -> bool: 

245 return name in self._elements 

246 

247 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None: 

248 """Return the `DimensionElement` with the given name or a default. 

249 

250 Parameters 

251 ---------- 

252 name : `str` 

253 Name of the element. 

254 default : `DimensionElement`, optional 

255 Element to return if the named one does not exist. Defaults to 

256 `None`. 

257 

258 Returns 

259 ------- 

260 element : `DimensionElement` 

261 The named element. 

262 """ 

263 return self._elements.get(name, default) 

264 

265 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

266 """Return a set of all static elements in this universe. 

267 

268 Non-static elements that are created as needed may also exist, but 

269 these are guaranteed to have no direct relationships to other elements 

270 (though they may have spatial or temporal relationships). 

271 

272 Returns 

273 ------- 

274 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

275 A frozen set of `DimensionElement` instances. 

276 """ 

277 return self._elements 

278 

279 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

280 """Return a set of all static dimensions in this universe. 

281 

282 Non-static dimensions that are created as needed may also exist, but 

283 these are guaranteed to have no direct relationships to other elements 

284 (though they may have spatial or temporal relationships). 

285 

286 Returns 

287 ------- 

288 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

289 A frozen set of `Dimension` instances. 

290 """ 

291 return self._dimensions 

292 

293 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

294 """Return a set of all `GovernorDimension` instances in this universe. 

295 

296 Returns 

297 ------- 

298 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

299 A frozen set of `GovernorDimension` instances. 

300 """ 

301 return self.governor_dimensions 

302 

303 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

304 """Return set of all `DatabaseDimensionElement` instances in universe. 

305 

306 This does not include `GovernorDimension` instances, which are backed 

307 by the database but do not inherit from `DatabaseDimensionElement`. 

308 

309 Returns 

310 ------- 

311 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

312 A frozen set of `DatabaseDimensionElement` instances. 

313 """ 

314 return self.database_elements 

315 

316 @property 

317 def elements(self) -> NamedValueAbstractSet[DimensionElement]: 

318 """All dimension elements defined in this universe.""" 

319 return self._elements 

320 

321 @property 

322 def dimensions(self) -> NamedValueAbstractSet[Dimension]: 

323 """All dimensions defined in this universe.""" 

324 return self._dimensions 

325 

326 @property 

327 @cached_getter 

328 def governor_dimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

329 """All governor dimensions defined in this universe. 

330 

331 Governor dimensions serve as special required dependencies of other 

332 dimensions, with special handling in dimension query expressions and 

333 collection summaries. Governor dimension records are stored in the 

334 database but the set of such values is expected to be small enough 

335 for all values to be cached by all clients. 

336 """ 

337 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

338 

339 @property 

340 @cached_getter 

341 def skypix_dimensions(self) -> NamedValueAbstractSet[SkyPixDimension]: 

342 """All skypix dimensions defined in this universe. 

343 

344 Skypix dimension records are always generated on-the-fly rather than 

345 stored in the database, and they always represent a tiling of the sky 

346 with no overlaps. 

347 """ 

348 result = NamedValueSet[SkyPixDimension]() 

349 for system in self.skypix: 

350 result.update(system) 

351 return result.freeze() 

352 

353 @property 

354 @cached_getter 

355 def database_elements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

356 """All dimension elements whose records are stored in the database, 

357 except governor dimensions. 

358 """ 

359 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

360 

361 @property 

362 @cached_getter 

363 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

364 """All skypix systems known to this universe. 

365 

366 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

367 """ 

368 return NamedValueSet( 

369 [ 

370 family 

371 for family in self._topology[TopologicalSpace.SPATIAL] 

372 if isinstance(family, SkyPixSystem) 

373 ] 

374 ).freeze() 

375 

376 def getElementIndex(self, name: str) -> int: 

377 """Return the position of the named dimension element. 

378 

379 The position is in this universe's sorting of all elements. 

380 

381 Parameters 

382 ---------- 

383 name : `str` 

384 Name of the element. 

385 

386 Returns 

387 ------- 

388 index : `int` 

389 Sorting index for this element. 

390 """ 

391 return self._elementIndices[name] 

392 

393 def getDimensionIndex(self, name: str) -> int: 

394 """Return the position of the named dimension. 

395 

396 This position is in this universe's sorting of all dimensions. 

397 

398 Parameters 

399 ---------- 

400 name : `str` 

401 Name of the dimension. 

402 

403 Returns 

404 ------- 

405 index : `int` 

406 Sorting index for this dimension. 

407 

408 Notes 

409 ----- 

410 The dimension sort order for a universe is consistent with the element 

411 order (all dimensions are elements), and either can be used to sort 

412 dimensions if used consistently. But there are also some contexts in 

413 which contiguous dimension-only indices are necessary or at least 

414 desirable. 

415 """ 

416 return self._dimensionIndices[name] 

417 

418 # TODO: remove on DM-41326. 

419 @deprecated( 

420 "Deprecated in favor of DimensionUniverse.conform, and will be removed after v27.", 

421 version="v27", 

422 category=FutureWarning, 

423 ) 

424 def expandDimensionNameSet(self, names: set[str]) -> None: 

425 """Expand a set of dimension names in-place. 

426 

427 Includes recursive dependencies. 

428 

429 This is an advanced interface for cases where constructing a 

430 `DimensionGraph` (which also expands required dependencies) is 

431 impossible or undesirable. 

432 

433 Parameters 

434 ---------- 

435 names : `set` [ `str` ] 

436 A true `set` of dimension names, to be expanded in-place. 

437 """ 

438 # Keep iterating until the set of names stops growing. This is not as 

439 # efficient as it could be, but we work pretty hard cache 

440 # DimensionGraph instances to keep actual construction rare, so that 

441 # shouldn't matter. 

442 oldSize = len(names) 

443 while True: 

444 # iterate over a temporary copy so we can modify the original 

445 for name in tuple(names): 

446 names.update(self._dimensions[name].required.names) 

447 names.update(self._dimensions[name].implied.names) 

448 if oldSize == len(names): 

449 break 

450 else: 

451 oldSize = len(names) 

452 

453 # TODO: remove on DM-41326. 

454 @deprecated( 

455 "DimensionUniverse.extract and DimensionGraph are deprecated in favor of DimensionUniverse.conform " 

456 "and DimensionGroup, and will be removed after v27.", 

457 version="v27", 

458 category=FutureWarning, 

459 ) 

460 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph: 

461 """Construct graph from iterable. 

462 

463 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

464 of `Dimension` instances and string names thereof. 

465 

466 Constructing `DimensionGraph` directly from names or dimension 

467 instances is slightly more efficient when it is known in advance that 

468 the iterable is not heterogenous. 

469 

470 Parameters 

471 ---------- 

472 iterable : iterable of `Dimension` or `str` 

473 Dimensions that must be included in the returned graph (their 

474 dependencies will be as well). 

475 

476 Returns 

477 ------- 

478 graph : `DimensionGraph` 

479 A `DimensionGraph` instance containing all given dimensions. 

480 """ 

481 return self.conform(iterable)._as_graph() 

482 

483 def conform( 

484 self, 

485 dimensions: Iterable[str | Dimension] | str | DimensionElement | DimensionGroup | DimensionGraph, 

486 /, 

487 ) -> DimensionGroup: 

488 """Construct a dimension group from an iterable of dimension names. 

489 

490 Parameters 

491 ---------- 

492 dimensions : `~collections.abc.Iterable` [ `str` or `Dimension` ], \ 

493 `str`, `DimensionElement`, `DimensionGroup`, or \ 

494 `DimensionGraph` 

495 Dimensions that must be included in the returned group; their 

496 dependencies will be as well. Support for `Dimension`, 

497 `DimensionElement` and `DimensionGraph` objects is deprecated and 

498 will be removed after v27. Passing `DimensionGraph` objects will 

499 not yield a deprecation warning to allow non-deprecated methods and 

500 properties that return `DimensionGraph` objects to be passed 

501 though, since these will be changed to return `DimensionGroup` in 

502 the future. 

503 

504 Returns 

505 ------- 

506 group : `DimensionGroup` 

507 A `DimensionGroup` instance containing all given dimensions. 

508 """ 

509 match dimensions: 

510 case DimensionGroup(): 

511 return dimensions 

512 case DimensionGraph(): 

513 return dimensions.as_group() 

514 case DimensionElement() as d: 

515 return d.minimal_group 

516 case str() as name: 

517 return self[name].minimal_group 

518 case iterable: 

519 names: set[str] = {getattr(d, "name", cast(str, d)) for d in iterable} 

520 return DimensionGroup(self, names) 

521 

522 @overload 

523 def sorted(self, elements: Iterable[Dimension], *, reverse: bool = False) -> Sequence[Dimension]: 

524 ... 

525 

526 @overload 

527 def sorted( 

528 self, elements: Iterable[DimensionElement | str], *, reverse: bool = False 

529 ) -> Sequence[DimensionElement]: 

530 ... 

531 

532 def sorted(self, elements: Iterable[Any], *, reverse: bool = False) -> list[Any]: 

533 """Return a sorted version of the given iterable of dimension elements. 

534 

535 The universe's sort order is topological (an element's dependencies 

536 precede it), with an unspecified (but deterministic) approach to 

537 breaking ties. 

538 

539 Parameters 

540 ---------- 

541 elements : iterable of `DimensionElement` 

542 Elements to be sorted. 

543 reverse : `bool`, optional 

544 If `True`, sort in the opposite order. 

545 

546 Returns 

547 ------- 

548 sorted : `~collections.abc.Sequence` [ `Dimension` or \ 

549 `DimensionElement` ] 

550 A sorted sequence containing the same elements that were given. 

551 """ 

552 s = set(elements) 

553 result = [element for element in self._elements if element in s or element.name in s] 

554 if reverse: 

555 result.reverse() 

556 return result 

557 

558 def getEncodeLength(self) -> int: 

559 """Return encoded size of graph. 

560 

561 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

562 instances in this universe. 

563 

564 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

565 information. 

566 """ 

567 return math.ceil(len(self._dimensions) / 8) 

568 

569 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]: 

570 """Return the set of `DimensionElement` objects whose 

571 `~DimensionElement.populated_by` attribute is the given dimension. 

572 

573 Parameters 

574 ---------- 

575 dimension : `Dimension` 

576 The dimension of interest. 

577 

578 Returns 

579 ------- 

580 populated_by : `NamedValueAbstractSet` [ `DimensionElement` ] 

581 The set of elements who say they are populated by the given 

582 dimension. 

583 """ 

584 return self._populates[dimension.name] 

585 

586 @property 

587 def empty(self) -> DimensionGraph: 

588 """The `DimensionGraph` that contains no dimensions. 

589 

590 After v27 this will be a `DimensionGroup`. 

591 """ 

592 return self._empty._as_graph() 

593 

594 @classmethod 

595 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse: 

596 """Return an unpickled dimension universe. 

597 

598 Callable used for unpickling. 

599 

600 For internal use only. 

601 """ 

602 if namespace is None: 

603 # Old pickled universe. 

604 namespace = _DEFAULT_NAMESPACE 

605 try: 

606 return cls._instances[version, namespace] 

607 except KeyError as err: 

608 raise pickle.UnpicklingError( 

609 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

610 "not found. Note that DimensionUniverse objects are not " 

611 "truly serialized; when using pickle to transfer them " 

612 "between processes, an equivalent instance with the same " 

613 "version must already exist in the receiving process." 

614 ) from err 

615 

616 def __reduce__(self) -> tuple: 

617 return (self._unpickle, (self._version, self._namespace)) 

618 

619 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

620 # DimensionUniverse is recursively immutable; see note in @immutable 

621 # decorator. 

622 return self 

623 

624 # Class attributes below are shadowed by instance attributes, and are 

625 # present just to hold the docstrings for those instance attributes. 

626 

627 commonSkyPix: SkyPixDimension 

628 """The special skypix dimension that is used to relate all other spatial 

629 dimensions in the `Registry` database (`SkyPixDimension`). 

630 """ 

631 

632 dimensionConfig: DimensionConfig 

633 """The configuration used to create this Universe (`DimensionConfig`).""" 

634 

635 _cached_groups: dict[frozenset[str], DimensionGroup] 

636 

637 _dimensions: NamedValueAbstractSet[Dimension] 

638 

639 _elements: NamedValueAbstractSet[DimensionElement] 

640 

641 _empty: DimensionGroup 

642 

643 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

644 

645 _dimensionIndices: dict[str, int] 

646 

647 _elementIndices: dict[str, int] 

648 

649 _populates: defaultdict[str, NamedValueSet[DimensionElement]] 

650 

651 _version: int 

652 

653 _namespace: str