Coverage for python/lsst/daf/butler/dimensions/_universe.py: 45%
197 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["DimensionUniverse"]
32import logging
33import math
34import pickle
35from collections import defaultdict
36from collections.abc import Iterable, Mapping, Sequence
37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, overload
39from deprecated.sphinx import deprecated
40from lsst.utils.classes import cached_getter, immutable
42from .._config import Config
43from .._named import NamedValueAbstractSet, NamedValueSet
44from .._topology import TopologicalFamily, TopologicalSpace
45from ._config import _DEFAULT_NAMESPACE, DimensionConfig
46from ._database import DatabaseDimensionElement
47from ._elements import Dimension, DimensionElement
48from ._governor import GovernorDimension
49from ._graph import DimensionGraph
50from ._group import DimensionGroup
51from ._skypix import SkyPixDimension, SkyPixSystem
53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
54 from .construction import DimensionConstructionBuilder
57E = TypeVar("E", bound=DimensionElement)
58_LOG = logging.getLogger(__name__)
61@immutable
62class DimensionUniverse: # numpydoc ignore=PR02
63 """Self-consistent set of dimensions.
65 A parent class that represents a complete, self-consistent set of
66 dimensions and their relationships.
68 `DimensionUniverse` is not a class-level singleton, but all instances are
69 tracked in a singleton map keyed by the version number and namespace
70 in the configuration they were loaded from. Because these universes
71 are solely responsible for constructing `DimensionElement` instances,
72 these are also indirectly tracked by that singleton as well.
74 Parameters
75 ----------
76 config : `Config`, optional
77 Configuration object from which dimension definitions can be extracted.
78 Ignored if ``builder`` is provided, or if ``version`` is provided and
79 an instance with that version already exists.
80 version : `int`, optional
81 Integer version for this `DimensionUniverse`. If not provided, a
82 version will be obtained from ``builder`` or ``config``.
83 namespace : `str`, optional
84 Namespace of this `DimensionUniverse`, combined with the version
85 to provide universe safety for registries that use different
86 dimension definitions.
87 builder : `DimensionConstructionBuilder`, optional
88 Builder object used to initialize a new instance. Ignored if
89 ``version`` is provided and an instance with that version already
90 exists. Should not have had `~DimensionConstructionBuilder.finish`
91 called; this will be called if needed by `DimensionUniverse`.
92 """
94 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {}
95 """Singleton dictionary of all instances, keyed by version.
97 For internal use only.
98 """
100 def __new__(
101 cls,
102 config: Config | None = None,
103 *,
104 version: int | None = None,
105 namespace: str | None = None,
106 builder: DimensionConstructionBuilder | None = None,
107 ) -> DimensionUniverse:
108 # Try to get a version first, to look for existing instances; try to
109 # do as little work as possible at this stage.
110 if version is None:
111 if builder is None:
112 config = DimensionConfig(config)
113 version = config["version"]
114 else:
115 version = builder.version
117 # Then a namespace.
118 if namespace is None:
119 if builder is None:
120 config = DimensionConfig(config)
121 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
122 else:
123 namespace = builder.namespace
124 # if still None use the default
125 if namespace is None:
126 namespace = _DEFAULT_NAMESPACE
128 # See if an equivalent instance already exists.
129 self: DimensionUniverse | None = cls._instances.get((version, namespace))
130 if self is not None:
131 return self
133 # Ensure we have a builder, building one from config if necessary.
134 if builder is None:
135 config = DimensionConfig(config)
136 builder = config.makeBuilder()
138 # Delegate to the builder for most of the construction work.
139 builder.finish()
141 # Create the universe instance and create core attributes, mostly
142 # copying from builder.
143 self = object.__new__(cls)
144 assert self is not None
145 self._cached_groups = {}
146 self._dimensions = builder.dimensions
147 self._elements = builder.elements
148 self._topology = builder.topology
149 self.dimensionConfig = builder.config
150 commonSkyPix = self._dimensions[builder.commonSkyPixName]
151 assert isinstance(commonSkyPix, SkyPixDimension)
152 self.commonSkyPix = commonSkyPix
154 # Attach self to all elements.
155 for element in self._elements:
156 element.universe = self
158 # Add attribute for special subsets of the graph.
159 self._empty = DimensionGroup(self, (), _conform=False)
161 # Use the version number and namespace from the config as a key in
162 # the singleton dict containing all instances; that will let us
163 # transfer dimension objects between processes using pickle without
164 # actually going through real initialization, as long as a universe
165 # with the same version and namespace has already been constructed in
166 # the receiving process.
167 self._version = version
168 self._namespace = namespace
169 cls._instances[self._version, self._namespace] = self
171 # Build mappings from element to index. These are used for
172 # topological-sort comparison operators in DimensionElement itself.
173 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
174 # Same for dimension to index, sorted topologically across required
175 # and implied. This is used for encode/decode.
176 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
178 self._populates = defaultdict(NamedValueSet)
179 for element in self._elements:
180 if element.populated_by is not None:
181 self._populates[element.populated_by.name].add(element)
183 return self
185 @property
186 def version(self) -> int:
187 """The version number of this universe.
189 Returns
190 -------
191 version : `int`
192 An integer representing the version number of this universe.
193 Uniquely defined when combined with the `namespace`.
194 """
195 return self._version
197 @property
198 def namespace(self) -> str:
199 """The namespace associated with this universe.
201 Returns
202 -------
203 namespace : `str`
204 The namespace. When combined with the `version` can uniquely
205 define this universe.
206 """
207 return self._namespace
209 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
210 """Check compatibility between this `DimensionUniverse` and another.
212 Parameters
213 ----------
214 other : `DimensionUniverse`
215 The other `DimensionUniverse` to check for compatibility.
217 Returns
218 -------
219 results : `bool`
220 If the other `DimensionUniverse` is compatible with this one return
221 `True`, else `False`.
222 """
223 # Different namespaces mean that these universes cannot be compatible.
224 if self.namespace != other.namespace:
225 return False
226 if self.version != other.version:
227 _LOG.info(
228 "Universes share a namespace %r but have differing versions (%d != %d). "
229 " This could be okay but may be responsible for dimension errors later.",
230 self.namespace,
231 self.version,
232 other.version,
233 )
235 # For now assume compatibility if versions differ.
236 return True
238 def __repr__(self) -> str:
239 return f"DimensionUniverse({self._version}, {self._namespace})"
241 def __getitem__(self, name: str) -> DimensionElement:
242 return self._elements[name]
244 def __contains__(self, name: Any) -> bool:
245 return name in self._elements
247 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None:
248 """Return the `DimensionElement` with the given name or a default.
250 Parameters
251 ----------
252 name : `str`
253 Name of the element.
254 default : `DimensionElement`, optional
255 Element to return if the named one does not exist. Defaults to
256 `None`.
258 Returns
259 -------
260 element : `DimensionElement`
261 The named element.
262 """
263 return self._elements.get(name, default)
265 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
266 """Return a set of all static elements in this universe.
268 Non-static elements that are created as needed may also exist, but
269 these are guaranteed to have no direct relationships to other elements
270 (though they may have spatial or temporal relationships).
272 Returns
273 -------
274 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
275 A frozen set of `DimensionElement` instances.
276 """
277 return self._elements
279 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
280 """Return a set of all static dimensions in this universe.
282 Non-static dimensions that are created as needed may also exist, but
283 these are guaranteed to have no direct relationships to other elements
284 (though they may have spatial or temporal relationships).
286 Returns
287 -------
288 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
289 A frozen set of `Dimension` instances.
290 """
291 return self._dimensions
293 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
294 """Return a set of all `GovernorDimension` instances in this universe.
296 Returns
297 -------
298 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
299 A frozen set of `GovernorDimension` instances.
300 """
301 return self.governor_dimensions
303 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
304 """Return set of all `DatabaseDimensionElement` instances in universe.
306 This does not include `GovernorDimension` instances, which are backed
307 by the database but do not inherit from `DatabaseDimensionElement`.
309 Returns
310 -------
311 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
312 A frozen set of `DatabaseDimensionElement` instances.
313 """
314 return self.database_elements
316 @property
317 def elements(self) -> NamedValueAbstractSet[DimensionElement]:
318 """All dimension elements defined in this universe."""
319 return self._elements
321 @property
322 def dimensions(self) -> NamedValueAbstractSet[Dimension]:
323 """All dimensions defined in this universe."""
324 return self._dimensions
326 @property
327 @cached_getter
328 def governor_dimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
329 """All governor dimensions defined in this universe.
331 Governor dimensions serve as special required dependencies of other
332 dimensions, with special handling in dimension query expressions and
333 collection summaries. Governor dimension records are stored in the
334 database but the set of such values is expected to be small enough
335 for all values to be cached by all clients.
336 """
337 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
339 @property
340 @cached_getter
341 def skypix_dimensions(self) -> NamedValueAbstractSet[SkyPixDimension]:
342 """All skypix dimensions defined in this universe.
344 Skypix dimension records are always generated on-the-fly rather than
345 stored in the database, and they always represent a tiling of the sky
346 with no overlaps.
347 """
348 result = NamedValueSet[SkyPixDimension]()
349 for system in self.skypix:
350 result.update(system)
351 return result.freeze()
353 @property
354 @cached_getter
355 def database_elements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
356 """All dimension elements whose records are stored in the database,
357 except governor dimensions.
358 """
359 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
361 @property
362 @cached_getter
363 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
364 """All skypix systems known to this universe.
366 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
367 """
368 return NamedValueSet(
369 [
370 family
371 for family in self._topology[TopologicalSpace.SPATIAL]
372 if isinstance(family, SkyPixSystem)
373 ]
374 ).freeze()
376 def getElementIndex(self, name: str) -> int:
377 """Return the position of the named dimension element.
379 The position is in this universe's sorting of all elements.
381 Parameters
382 ----------
383 name : `str`
384 Name of the element.
386 Returns
387 -------
388 index : `int`
389 Sorting index for this element.
390 """
391 return self._elementIndices[name]
393 def getDimensionIndex(self, name: str) -> int:
394 """Return the position of the named dimension.
396 This position is in this universe's sorting of all dimensions.
398 Parameters
399 ----------
400 name : `str`
401 Name of the dimension.
403 Returns
404 -------
405 index : `int`
406 Sorting index for this dimension.
408 Notes
409 -----
410 The dimension sort order for a universe is consistent with the element
411 order (all dimensions are elements), and either can be used to sort
412 dimensions if used consistently. But there are also some contexts in
413 which contiguous dimension-only indices are necessary or at least
414 desirable.
415 """
416 return self._dimensionIndices[name]
418 # TODO: remove on DM-41326.
419 @deprecated(
420 "Deprecated in favor of DimensionUniverse.conform, and will be removed after v27.",
421 version="v27",
422 category=FutureWarning,
423 )
424 def expandDimensionNameSet(self, names: set[str]) -> None:
425 """Expand a set of dimension names in-place.
427 Includes recursive dependencies.
429 This is an advanced interface for cases where constructing a
430 `DimensionGraph` (which also expands required dependencies) is
431 impossible or undesirable.
433 Parameters
434 ----------
435 names : `set` [ `str` ]
436 A true `set` of dimension names, to be expanded in-place.
437 """
438 # Keep iterating until the set of names stops growing. This is not as
439 # efficient as it could be, but we work pretty hard cache
440 # DimensionGraph instances to keep actual construction rare, so that
441 # shouldn't matter.
442 oldSize = len(names)
443 while True:
444 # iterate over a temporary copy so we can modify the original
445 for name in tuple(names):
446 names.update(self._dimensions[name].required.names)
447 names.update(self._dimensions[name].implied.names)
448 if oldSize == len(names):
449 break
450 else:
451 oldSize = len(names)
453 # TODO: remove on DM-41326.
454 @deprecated(
455 "DimensionUniverse.extract and DimensionGraph are deprecated in favor of DimensionUniverse.conform "
456 "and DimensionGroup, and will be removed after v27.",
457 version="v27",
458 category=FutureWarning,
459 )
460 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph:
461 """Construct graph from iterable.
463 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
464 of `Dimension` instances and string names thereof.
466 Constructing `DimensionGraph` directly from names or dimension
467 instances is slightly more efficient when it is known in advance that
468 the iterable is not heterogenous.
470 Parameters
471 ----------
472 iterable : iterable of `Dimension` or `str`
473 Dimensions that must be included in the returned graph (their
474 dependencies will be as well).
476 Returns
477 -------
478 graph : `DimensionGraph`
479 A `DimensionGraph` instance containing all given dimensions.
480 """
481 return self.conform(iterable)._as_graph()
483 def conform(
484 self,
485 dimensions: Iterable[str | Dimension] | str | DimensionElement | DimensionGroup | DimensionGraph,
486 /,
487 ) -> DimensionGroup:
488 """Construct a dimension group from an iterable of dimension names.
490 Parameters
491 ----------
492 dimensions : `~collections.abc.Iterable` [ `str` or `Dimension` ], \
493 `str`, `DimensionElement`, `DimensionGroup`, or \
494 `DimensionGraph`
495 Dimensions that must be included in the returned group; their
496 dependencies will be as well. Support for `Dimension`,
497 `DimensionElement` and `DimensionGraph` objects is deprecated and
498 will be removed after v27. Passing `DimensionGraph` objects will
499 not yield a deprecation warning to allow non-deprecated methods and
500 properties that return `DimensionGraph` objects to be passed
501 though, since these will be changed to return `DimensionGroup` in
502 the future.
504 Returns
505 -------
506 group : `DimensionGroup`
507 A `DimensionGroup` instance containing all given dimensions.
508 """
509 match dimensions:
510 case DimensionGroup():
511 return dimensions
512 case DimensionGraph():
513 return dimensions.as_group()
514 case DimensionElement() as d:
515 return d.minimal_group
516 case str() as name:
517 return self[name].minimal_group
518 case iterable:
519 names: set[str] = {getattr(d, "name", cast(str, d)) for d in iterable}
520 return DimensionGroup(self, names)
522 @overload
523 def sorted(self, elements: Iterable[Dimension], *, reverse: bool = False) -> Sequence[Dimension]:
524 ...
526 @overload
527 def sorted(
528 self, elements: Iterable[DimensionElement | str], *, reverse: bool = False
529 ) -> Sequence[DimensionElement]:
530 ...
532 def sorted(self, elements: Iterable[Any], *, reverse: bool = False) -> list[Any]:
533 """Return a sorted version of the given iterable of dimension elements.
535 The universe's sort order is topological (an element's dependencies
536 precede it), with an unspecified (but deterministic) approach to
537 breaking ties.
539 Parameters
540 ----------
541 elements : iterable of `DimensionElement`
542 Elements to be sorted.
543 reverse : `bool`, optional
544 If `True`, sort in the opposite order.
546 Returns
547 -------
548 sorted : `~collections.abc.Sequence` [ `Dimension` or \
549 `DimensionElement` ]
550 A sorted sequence containing the same elements that were given.
551 """
552 s = set(elements)
553 result = [element for element in self._elements if element in s or element.name in s]
554 if reverse:
555 result.reverse()
556 return result
558 def getEncodeLength(self) -> int:
559 """Return encoded size of graph.
561 Returns the size (in bytes) of the encoded size of `DimensionGraph`
562 instances in this universe.
564 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
565 information.
566 """
567 return math.ceil(len(self._dimensions) / 8)
569 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]:
570 """Return the set of `DimensionElement` objects whose
571 `~DimensionElement.populated_by` attribute is the given dimension.
573 Parameters
574 ----------
575 dimension : `Dimension`
576 The dimension of interest.
578 Returns
579 -------
580 populated_by : `NamedValueAbstractSet` [ `DimensionElement` ]
581 The set of elements who say they are populated by the given
582 dimension.
583 """
584 return self._populates[dimension.name]
586 @property
587 def empty(self) -> DimensionGraph:
588 """The `DimensionGraph` that contains no dimensions.
590 After v27 this will be a `DimensionGroup`.
591 """
592 return self._empty._as_graph()
594 @classmethod
595 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse:
596 """Return an unpickled dimension universe.
598 Callable used for unpickling.
600 For internal use only.
601 """
602 if namespace is None:
603 # Old pickled universe.
604 namespace = _DEFAULT_NAMESPACE
605 try:
606 return cls._instances[version, namespace]
607 except KeyError as err:
608 raise pickle.UnpicklingError(
609 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
610 "not found. Note that DimensionUniverse objects are not "
611 "truly serialized; when using pickle to transfer them "
612 "between processes, an equivalent instance with the same "
613 "version must already exist in the receiving process."
614 ) from err
616 def __reduce__(self) -> tuple:
617 return (self._unpickle, (self._version, self._namespace))
619 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
620 # DimensionUniverse is recursively immutable; see note in @immutable
621 # decorator.
622 return self
624 # Class attributes below are shadowed by instance attributes, and are
625 # present just to hold the docstrings for those instance attributes.
627 commonSkyPix: SkyPixDimension
628 """The special skypix dimension that is used to relate all other spatial
629 dimensions in the `Registry` database (`SkyPixDimension`).
630 """
632 dimensionConfig: DimensionConfig
633 """The configuration used to create this Universe (`DimensionConfig`)."""
635 _cached_groups: dict[frozenset[str], DimensionGroup]
637 _dimensions: NamedValueAbstractSet[Dimension]
639 _elements: NamedValueAbstractSet[DimensionElement]
641 _empty: DimensionGroup
643 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
645 _dimensionIndices: dict[str, int]
647 _elementIndices: dict[str, int]
649 _populates: defaultdict[str, NamedValueSet[DimensionElement]]
651 _version: int
653 _namespace: str