Coverage for python/lsst/daf/butler/dimensions/_universe.py: 46%
195 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 03:16 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 03:16 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["DimensionUniverse"]
32import logging
33import math
34import pickle
35from collections import defaultdict
36from collections.abc import Iterable, Mapping, Sequence
37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, overload
39from deprecated.sphinx import deprecated
40from lsst.utils.classes import cached_getter, immutable
42from .._config import Config
43from .._named import NamedValueAbstractSet, NamedValueSet
44from .._topology import TopologicalFamily, TopologicalSpace
45from .._utilities.thread_safe_cache import ThreadSafeCache
46from ._config import _DEFAULT_NAMESPACE, DimensionConfig
47from ._database import DatabaseDimensionElement
48from ._elements import Dimension, DimensionElement
49from ._governor import GovernorDimension
50from ._graph import DimensionGraph
51from ._group import DimensionGroup
52from ._skypix import SkyPixDimension, SkyPixSystem
54if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
55 from .construction import DimensionConstructionBuilder
58E = TypeVar("E", bound=DimensionElement)
59_LOG = logging.getLogger(__name__)
62@immutable
63class DimensionUniverse: # numpydoc ignore=PR02
64 """Self-consistent set of dimensions.
66 A parent class that represents a complete, self-consistent set of
67 dimensions and their relationships.
69 `DimensionUniverse` is not a class-level singleton, but all instances are
70 tracked in a singleton map keyed by the version number and namespace
71 in the configuration they were loaded from. Because these universes
72 are solely responsible for constructing `DimensionElement` instances,
73 these are also indirectly tracked by that singleton as well.
75 Parameters
76 ----------
77 config : `Config`, optional
78 Configuration object from which dimension definitions can be extracted.
79 Ignored if ``builder`` is provided, or if ``version`` is provided and
80 an instance with that version already exists.
81 version : `int`, optional
82 Integer version for this `DimensionUniverse`. If not provided, a
83 version will be obtained from ``builder`` or ``config``.
84 namespace : `str`, optional
85 Namespace of this `DimensionUniverse`, combined with the version
86 to provide universe safety for registries that use different
87 dimension definitions.
88 builder : `DimensionConstructionBuilder`, optional
89 Builder object used to initialize a new instance. Ignored if
90 ``version`` is provided and an instance with that version already
91 exists. Should not have had `~DimensionConstructionBuilder.finish`
92 called; this will be called if needed by `DimensionUniverse`.
93 """
95 _instances: ClassVar[ThreadSafeCache[tuple[int, str], DimensionUniverse]] = ThreadSafeCache()
96 """Singleton dictionary of all instances, keyed by version.
98 For internal use only.
99 """
101 def __new__(
102 cls,
103 config: Config | None = None,
104 *,
105 version: int | None = None,
106 namespace: str | None = None,
107 builder: DimensionConstructionBuilder | None = None,
108 ) -> DimensionUniverse:
109 # Try to get a version first, to look for existing instances; try to
110 # do as little work as possible at this stage.
111 if version is None:
112 if builder is None:
113 config = DimensionConfig(config)
114 version = config["version"]
115 else:
116 version = builder.version
118 # Then a namespace.
119 if namespace is None:
120 if builder is None:
121 config = DimensionConfig(config)
122 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
123 else:
124 namespace = builder.namespace
125 # if still None use the default
126 if namespace is None:
127 namespace = _DEFAULT_NAMESPACE
129 # See if an equivalent instance already exists.
130 self: DimensionUniverse | None = cls._instances.get((version, namespace))
131 if self is not None:
132 return self
134 # Ensure we have a builder, building one from config if necessary.
135 if builder is None:
136 config = DimensionConfig(config)
137 builder = config.makeBuilder()
139 # Delegate to the builder for most of the construction work.
140 builder.finish()
142 # Create the universe instance and create core attributes, mostly
143 # copying from builder.
144 self = object.__new__(cls)
145 assert self is not None
146 self._cached_groups = ThreadSafeCache()
147 self._dimensions = builder.dimensions
148 self._elements = builder.elements
149 self._topology = builder.topology
150 self.dimensionConfig = builder.config
151 commonSkyPix = self._dimensions[builder.commonSkyPixName]
152 assert isinstance(commonSkyPix, SkyPixDimension)
153 self.commonSkyPix = commonSkyPix
155 # Attach self to all elements.
156 for element in self._elements:
157 element.universe = self
159 # Add attribute for special subsets of the graph.
160 self._empty = DimensionGroup(self, (), _conform=False)
162 # Use the version number and namespace from the config as a key in
163 # the singleton dict containing all instances; that will let us
164 # transfer dimension objects between processes using pickle without
165 # actually going through real initialization, as long as a universe
166 # with the same version and namespace has already been constructed in
167 # the receiving process.
168 self._version = version
169 self._namespace = namespace
171 # Build mappings from element to index. These are used for
172 # topological-sort comparison operators in DimensionElement itself.
173 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
174 # Same for dimension to index, sorted topologically across required
175 # and implied. This is used for encode/decode.
176 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
178 self._populates = defaultdict(NamedValueSet)
179 for element in self._elements:
180 if element.populated_by is not None:
181 self._populates[element.populated_by.name].add(element)
183 return cls._instances.set_or_get((self._version, self._namespace), self)
185 @property
186 def version(self) -> int:
187 """The version number of this universe.
189 Returns
190 -------
191 version : `int`
192 An integer representing the version number of this universe.
193 Uniquely defined when combined with the `namespace`.
194 """
195 return self._version
197 @property
198 def namespace(self) -> str:
199 """The namespace associated with this universe.
201 Returns
202 -------
203 namespace : `str`
204 The namespace. When combined with the `version` can uniquely
205 define this universe.
206 """
207 return self._namespace
209 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
210 """Check compatibility between this `DimensionUniverse` and another.
212 Parameters
213 ----------
214 other : `DimensionUniverse`
215 The other `DimensionUniverse` to check for compatibility.
217 Returns
218 -------
219 results : `bool`
220 If the other `DimensionUniverse` is compatible with this one return
221 `True`, else `False`.
222 """
223 # Different namespaces mean that these universes cannot be compatible.
224 if self.namespace != other.namespace:
225 return False
226 if self.version != other.version:
227 _LOG.info(
228 "Universes share a namespace %r but have differing versions (%d != %d). "
229 " This could be okay but may be responsible for dimension errors later.",
230 self.namespace,
231 self.version,
232 other.version,
233 )
235 # For now assume compatibility if versions differ.
236 return True
238 def __repr__(self) -> str:
239 return f"DimensionUniverse({self._version}, {self._namespace})"
241 def __getitem__(self, name: str) -> DimensionElement:
242 return self._elements[name]
244 def __contains__(self, name: Any) -> bool:
245 return name in self._elements
247 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None:
248 """Return the `DimensionElement` with the given name or a default.
250 Parameters
251 ----------
252 name : `str`
253 Name of the element.
254 default : `DimensionElement`, optional
255 Element to return if the named one does not exist. Defaults to
256 `None`.
258 Returns
259 -------
260 element : `DimensionElement`
261 The named element.
262 """
263 return self._elements.get(name, default)
265 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
266 """Return a set of all static elements in this universe.
268 Non-static elements that are created as needed may also exist, but
269 these are guaranteed to have no direct relationships to other elements
270 (though they may have spatial or temporal relationships).
272 Returns
273 -------
274 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
275 A frozen set of `DimensionElement` instances.
276 """
277 return self._elements
279 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
280 """Return a set of all static dimensions in this universe.
282 Non-static dimensions that are created as needed may also exist, but
283 these are guaranteed to have no direct relationships to other elements
284 (though they may have spatial or temporal relationships).
286 Returns
287 -------
288 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
289 A frozen set of `Dimension` instances.
290 """
291 return self._dimensions
293 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
294 """Return a set of all `GovernorDimension` instances in this universe.
296 Returns
297 -------
298 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
299 A frozen set of `GovernorDimension` instances.
300 """
301 return self.governor_dimensions
303 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
304 """Return set of all `DatabaseDimensionElement` instances in universe.
306 This does not include `GovernorDimension` instances, which are backed
307 by the database but do not inherit from `DatabaseDimensionElement`.
309 Returns
310 -------
311 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
312 A frozen set of `DatabaseDimensionElement` instances.
313 """
314 return self.database_elements
316 @property
317 def elements(self) -> NamedValueAbstractSet[DimensionElement]:
318 """All dimension elements defined in this universe."""
319 return self._elements
321 @property
322 def dimensions(self) -> NamedValueAbstractSet[Dimension]:
323 """All dimensions defined in this universe."""
324 return self._dimensions
326 @property
327 @cached_getter
328 def governor_dimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
329 """All governor dimensions defined in this universe.
331 Governor dimensions serve as special required dependencies of other
332 dimensions, with special handling in dimension query expressions and
333 collection summaries. Governor dimension records are stored in the
334 database but the set of such values is expected to be small enough
335 for all values to be cached by all clients.
336 """
337 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
339 @property
340 @cached_getter
341 def skypix_dimensions(self) -> NamedValueAbstractSet[SkyPixDimension]:
342 """All skypix dimensions defined in this universe.
344 Skypix dimension records are always generated on-the-fly rather than
345 stored in the database, and they always represent a tiling of the sky
346 with no overlaps.
347 """
348 result = NamedValueSet[SkyPixDimension]()
349 for system in self.skypix:
350 result.update(system)
351 return result.freeze()
353 @property
354 @cached_getter
355 def database_elements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
356 """All dimension elements whose records are stored in the database,
357 except governor dimensions.
358 """
359 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
361 @property
362 @cached_getter
363 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
364 """All skypix systems known to this universe.
366 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
367 """
368 return NamedValueSet(
369 [
370 family
371 for family in self._topology[TopologicalSpace.SPATIAL]
372 if isinstance(family, SkyPixSystem)
373 ]
374 ).freeze()
376 def getElementIndex(self, name: str) -> int:
377 """Return the position of the named dimension element.
379 The position is in this universe's sorting of all elements.
381 Parameters
382 ----------
383 name : `str`
384 Name of the element.
386 Returns
387 -------
388 index : `int`
389 Sorting index for this element.
390 """
391 return self._elementIndices[name]
393 def getDimensionIndex(self, name: str) -> int:
394 """Return the position of the named dimension.
396 This position is in this universe's sorting of all dimensions.
398 Parameters
399 ----------
400 name : `str`
401 Name of the dimension.
403 Returns
404 -------
405 index : `int`
406 Sorting index for this dimension.
408 Notes
409 -----
410 The dimension sort order for a universe is consistent with the element
411 order (all dimensions are elements), and either can be used to sort
412 dimensions if used consistently. But there are also some contexts in
413 which contiguous dimension-only indices are necessary or at least
414 desirable.
415 """
416 return self._dimensionIndices[name]
418 # TODO: remove on DM-41326.
419 @deprecated(
420 "Deprecated in favor of DimensionUniverse.conform, and will be removed after v27.",
421 version="v27",
422 category=FutureWarning,
423 )
424 def expandDimensionNameSet(self, names: set[str]) -> None:
425 """Expand a set of dimension names in-place.
427 Includes recursive dependencies.
429 This is an advanced interface for cases where constructing a
430 `DimensionGraph` (which also expands required dependencies) is
431 impossible or undesirable.
433 Parameters
434 ----------
435 names : `set` [ `str` ]
436 A true `set` of dimension names, to be expanded in-place.
437 """
438 # Keep iterating until the set of names stops growing. This is not as
439 # efficient as it could be, but we work pretty hard cache
440 # DimensionGraph instances to keep actual construction rare, so that
441 # shouldn't matter.
442 oldSize = len(names)
443 while True:
444 # iterate over a temporary copy so we can modify the original
445 for name in tuple(names):
446 names.update(self._dimensions[name].required.names)
447 names.update(self._dimensions[name].implied.names)
448 if oldSize == len(names):
449 break
450 else:
451 oldSize = len(names)
453 # TODO: remove on DM-41326.
454 @deprecated(
455 "DimensionUniverse.extract and DimensionGraph are deprecated in favor of DimensionUniverse.conform "
456 "and DimensionGroup, and will be removed after v27.",
457 version="v27",
458 category=FutureWarning,
459 )
460 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph:
461 """Construct graph from iterable.
463 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
464 of `Dimension` instances and string names thereof.
466 Constructing `DimensionGraph` directly from names or dimension
467 instances is slightly more efficient when it is known in advance that
468 the iterable is not heterogenous.
470 Parameters
471 ----------
472 iterable : iterable of `Dimension` or `str`
473 Dimensions that must be included in the returned graph (their
474 dependencies will be as well).
476 Returns
477 -------
478 graph : `DimensionGraph`
479 A `DimensionGraph` instance containing all given dimensions.
480 """
481 return self.conform(iterable)._as_graph()
483 def conform(
484 self,
485 dimensions: Iterable[str | Dimension] | str | DimensionElement | DimensionGroup | DimensionGraph,
486 /,
487 ) -> DimensionGroup:
488 """Construct a dimension group from an iterable of dimension names.
490 Parameters
491 ----------
492 dimensions : `~collections.abc.Iterable` [ `str` or `Dimension` ], \
493 `str`, `DimensionElement`, `DimensionGroup`, or \
494 `DimensionGraph`
495 Dimensions that must be included in the returned group; their
496 dependencies will be as well. Support for `Dimension`,
497 `DimensionElement` and `DimensionGraph` objects is deprecated and
498 will be removed after v27. Passing `DimensionGraph` objects will
499 not yield a deprecation warning to allow non-deprecated methods and
500 properties that return `DimensionGraph` objects to be passed
501 though, since these will be changed to return `DimensionGroup` in
502 the future.
504 Returns
505 -------
506 group : `DimensionGroup`
507 A `DimensionGroup` instance containing all given dimensions.
508 """
509 match dimensions:
510 case DimensionGroup():
511 return dimensions
512 case DimensionGraph():
513 return dimensions.as_group()
514 case DimensionElement() as d:
515 return d.minimal_group
516 case str() as name:
517 return self[name].minimal_group
518 case iterable:
519 names: set[str] = {getattr(d, "name", cast(str, d)) for d in iterable}
520 return DimensionGroup(self, names)
522 @overload
523 def sorted(self, elements: Iterable[Dimension], *, reverse: bool = False) -> Sequence[Dimension]: ... 523 ↛ exitline 523 didn't return from function 'sorted', because
525 @overload
526 def sorted( 526 ↛ exitline 526 didn't jump to the function exit
527 self, elements: Iterable[DimensionElement | str], *, reverse: bool = False
528 ) -> Sequence[DimensionElement]: ...
530 def sorted(self, elements: Iterable[Any], *, reverse: bool = False) -> list[Any]:
531 """Return a sorted version of the given iterable of dimension elements.
533 The universe's sort order is topological (an element's dependencies
534 precede it), with an unspecified (but deterministic) approach to
535 breaking ties.
537 Parameters
538 ----------
539 elements : iterable of `DimensionElement`
540 Elements to be sorted.
541 reverse : `bool`, optional
542 If `True`, sort in the opposite order.
544 Returns
545 -------
546 sorted : `~collections.abc.Sequence` [ `Dimension` or \
547 `DimensionElement` ]
548 A sorted sequence containing the same elements that were given.
549 """
550 s = set(elements)
551 result = [element for element in self._elements if element in s or element.name in s]
552 if reverse:
553 result.reverse()
554 return result
556 def getEncodeLength(self) -> int:
557 """Return encoded size of graph.
559 Returns the size (in bytes) of the encoded size of `DimensionGraph`
560 instances in this universe.
562 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
563 information.
564 """
565 return math.ceil(len(self._dimensions) / 8)
567 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]:
568 """Return the set of `DimensionElement` objects whose
569 `~DimensionElement.populated_by` attribute is the given dimension.
571 Parameters
572 ----------
573 dimension : `Dimension`
574 The dimension of interest.
576 Returns
577 -------
578 populated_by : `NamedValueAbstractSet` [ `DimensionElement` ]
579 The set of elements who say they are populated by the given
580 dimension.
581 """
582 return self._populates[dimension.name]
584 @property
585 def empty(self) -> DimensionGraph:
586 """The `DimensionGraph` that contains no dimensions.
588 After v27 this will be a `DimensionGroup`.
589 """
590 return self._empty._as_graph()
592 @classmethod
593 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse:
594 """Return an unpickled dimension universe.
596 Callable used for unpickling.
598 For internal use only.
599 """
600 if namespace is None:
601 # Old pickled universe.
602 namespace = _DEFAULT_NAMESPACE
603 instance = cls._instances.get((version, namespace))
604 if instance is None:
605 raise pickle.UnpicklingError(
606 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
607 "not found. Note that DimensionUniverse objects are not "
608 "truly serialized; when using pickle to transfer them "
609 "between processes, an equivalent instance with the same "
610 "version must already exist in the receiving process."
611 )
612 return instance
614 def __reduce__(self) -> tuple:
615 return (self._unpickle, (self._version, self._namespace))
617 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
618 # DimensionUniverse is recursively immutable; see note in @immutable
619 # decorator.
620 return self
622 # Class attributes below are shadowed by instance attributes, and are
623 # present just to hold the docstrings for those instance attributes.
625 commonSkyPix: SkyPixDimension
626 """The special skypix dimension that is used to relate all other spatial
627 dimensions in the `Registry` database (`SkyPixDimension`).
628 """
630 dimensionConfig: DimensionConfig
631 """The configuration used to create this Universe (`DimensionConfig`)."""
633 _cached_groups: ThreadSafeCache[frozenset[str], DimensionGroup]
635 _dimensions: NamedValueAbstractSet[Dimension]
637 _elements: NamedValueAbstractSet[DimensionElement]
639 _empty: DimensionGroup
641 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
643 _dimensionIndices: dict[str, int]
645 _elementIndices: dict[str, int]
647 _populates: defaultdict[str, NamedValueSet[DimensionElement]]
649 _version: int
651 _namespace: str