Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 36%
156 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-06 02:34 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-06 02:34 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import logging
27import math
28import pickle
29from typing import (
30 TYPE_CHECKING,
31 Any,
32 ClassVar,
33 Dict,
34 FrozenSet,
35 Iterable,
36 List,
37 Mapping,
38 Optional,
39 Set,
40 Tuple,
41 TypeVar,
42 Union,
43)
45from deprecated.sphinx import deprecated
46from lsst.utils.classes import cached_getter, immutable
48from .._topology import TopologicalFamily, TopologicalSpace
49from ..config import Config
50from ..named import NamedValueAbstractSet, NamedValueSet
51from ._config import _DEFAULT_NAMESPACE, DimensionConfig
52from ._database import DatabaseDimensionElement
53from ._elements import Dimension, DimensionElement
54from ._governor import GovernorDimension
55from ._graph import DimensionGraph
56from ._skypix import SkyPixDimension, SkyPixSystem
58if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
59 from ._coordinate import DataCoordinate
60 from ._packer import DimensionPacker, DimensionPackerFactory
61 from .construction import DimensionConstructionBuilder
64E = TypeVar("E", bound=DimensionElement)
65_LOG = logging.getLogger(__name__)
68@immutable
69class DimensionUniverse:
70 """Self-consistent set of dimensions.
72 A parent class that represents a complete, self-consistent set of
73 dimensions and their relationships.
75 `DimensionUniverse` is not a class-level singleton, but all instances are
76 tracked in a singleton map keyed by the version number and namespace
77 in the configuration they were loaded from. Because these universes
78 are solely responsible for constructing `DimensionElement` instances,
79 these are also indirectly tracked by that singleton as well.
81 Parameters
82 ----------
83 config : `Config`, optional
84 Configuration object from which dimension definitions can be extracted.
85 Ignored if ``builder`` is provided, or if ``version`` is provided and
86 an instance with that version already exists.
87 version : `int`, optional
88 Integer version for this `DimensionUniverse`. If not provided, a
89 version will be obtained from ``builder`` or ``config``.
90 namespace : `str`, optional
91 Namespace of this `DimensionUniverse`, combined with the version
92 to provide universe safety for registries that use different
93 dimension definitions.
94 builder : `DimensionConstructionBuilder`, optional
95 Builder object used to initialize a new instance. Ignored if
96 ``version`` is provided and an instance with that version already
97 exists. Should not have had `~DimensionConstructionBuilder.finish`
98 called; this will be called if needed by `DimensionUniverse`.
99 """
101 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {}
102 """Singleton dictionary of all instances, keyed by version.
104 For internal use only.
105 """
107 def __new__(
108 cls,
109 config: Optional[Config] = None,
110 *,
111 version: Optional[int] = None,
112 namespace: Optional[str] = None,
113 builder: Optional[DimensionConstructionBuilder] = None,
114 ) -> DimensionUniverse:
115 # Try to get a version first, to look for existing instances; try to
116 # do as little work as possible at this stage.
117 if version is None:
118 if builder is None:
119 config = DimensionConfig(config)
120 version = config["version"]
121 else:
122 version = builder.version
124 # Then a namespace.
125 if namespace is None:
126 if builder is None:
127 config = DimensionConfig(config)
128 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
129 else:
130 namespace = builder.namespace
131 # if still None use the default
132 if namespace is None:
133 namespace = _DEFAULT_NAMESPACE
135 # See if an equivalent instance already exists.
136 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace))
137 if self is not None:
138 return self
140 # Ensure we have a builder, building one from config if necessary.
141 if builder is None:
142 config = DimensionConfig(config)
143 builder = config.makeBuilder()
145 # Delegate to the builder for most of the construction work.
146 builder.finish()
148 # Create the universe instance and create core attributes, mostly
149 # copying from builder.
150 self = object.__new__(cls)
151 assert self is not None
152 self._cache = {}
153 self._dimensions = builder.dimensions
154 self._elements = builder.elements
155 self._topology = builder.topology
156 self._packers = builder.packers
157 self.dimensionConfig = builder.config
158 commonSkyPix = self._dimensions[builder.commonSkyPixName]
159 assert isinstance(commonSkyPix, SkyPixDimension)
160 self.commonSkyPix = commonSkyPix
162 # Attach self to all elements.
163 for element in self._elements:
164 element.universe = self
166 # Add attribute for special subsets of the graph.
167 self.empty = DimensionGraph(self, (), conform=False)
169 # Use the version number and namespace from the config as a key in
170 # the singleton dict containing all instances; that will let us
171 # transfer dimension objects between processes using pickle without
172 # actually going through real initialization, as long as a universe
173 # with the same version and namespace has already been constructed in
174 # the receiving process.
175 self._version = version
176 self._namespace = namespace
177 cls._instances[self._version, self._namespace] = self
179 # Build mappings from element to index. These are used for
180 # topological-sort comparison operators in DimensionElement itself.
181 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
182 # Same for dimension to index, sorted topologically across required
183 # and implied. This is used for encode/decode.
184 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
186 return self
188 @property
189 def version(self) -> int:
190 """The version number of this universe.
192 Returns
193 -------
194 version : `int`
195 An integer representing the version number of this universe.
196 Uniquely defined when combined with the `namespace`.
197 """
198 return self._version
200 @property
201 def namespace(self) -> str:
202 """The namespace associated with this universe.
204 Returns
205 -------
206 namespace : `str`
207 The namespace. When combined with the `version` can uniquely
208 define this universe.
209 """
210 return self._namespace
212 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
213 """Check compatibility between this `DimensionUniverse` and another
215 Parameters
216 ----------
217 other : `DimensionUniverse`
218 The other `DimensionUniverse` to check for compatibility
220 Returns
221 -------
222 results : `bool`
223 If the other `DimensionUniverse` is compatible with this one return
224 `True`, else `False`
225 """
226 # Different namespaces mean that these universes cannot be compatible.
227 if self.namespace != other.namespace:
228 return False
229 if self.version != other.version:
230 _LOG.info(
231 "Universes share a namespace %r but have differing versions (%d != %d). "
232 " This could be okay but may be responsible for dimension errors later.",
233 self.namespace,
234 self.version,
235 other.version,
236 )
238 # For now assume compatibility if versions differ.
239 return True
241 def __repr__(self) -> str:
242 return f"DimensionUniverse({self._version}, {self._namespace})"
244 def __getitem__(self, name: str) -> DimensionElement:
245 return self._elements[name]
247 def __contains__(self, name: Any) -> bool:
248 return name in self._elements
250 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
251 """Return the `DimensionElement` with the given name or a default.
253 Parameters
254 ----------
255 name : `str`
256 Name of the element.
257 default : `DimensionElement`, optional
258 Element to return if the named one does not exist. Defaults to
259 `None`.
261 Returns
262 -------
263 element : `DimensionElement`
264 The named element.
265 """
266 return self._elements.get(name, default)
268 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
269 """Return a set of all static elements in this universe.
271 Non-static elements that are created as needed may also exist, but
272 these are guaranteed to have no direct relationships to other elements
273 (though they may have spatial or temporal relationships).
275 Returns
276 -------
277 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
278 A frozen set of `DimensionElement` instances.
279 """
280 return self._elements
282 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
283 """Return a set of all static dimensions in this universe.
285 Non-static dimensions that are created as needed may also exist, but
286 these are guaranteed to have no direct relationships to other elements
287 (though they may have spatial or temporal relationships).
289 Returns
290 -------
291 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
292 A frozen set of `Dimension` instances.
293 """
294 return self._dimensions
296 @cached_getter
297 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
298 """Return a set of all `GovernorDimension` instances in this universe.
300 Returns
301 -------
302 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
303 A frozen set of `GovernorDimension` instances.
304 """
305 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
307 @cached_getter
308 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
309 """Return set of all `DatabaseDimensionElement` instances in universe.
311 This does not include `GovernorDimension` instances, which are backed
312 by the database but do not inherit from `DatabaseDimensionElement`.
314 Returns
315 -------
316 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
317 A frozen set of `DatabaseDimensionElement` instances.
318 """
319 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
321 @property
322 @cached_getter
323 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
324 """All skypix systems known to this universe.
326 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
327 """
328 return NamedValueSet(
329 [
330 family
331 for family in self._topology[TopologicalSpace.SPATIAL]
332 if isinstance(family, SkyPixSystem)
333 ]
334 ).freeze()
336 def getElementIndex(self, name: str) -> int:
337 """Return the position of the named dimension element.
339 The position is in this universe's sorting of all elements.
341 Parameters
342 ----------
343 name : `str`
344 Name of the element.
346 Returns
347 -------
348 index : `int`
349 Sorting index for this element.
350 """
351 return self._elementIndices[name]
353 def getDimensionIndex(self, name: str) -> int:
354 """Return the position of the named dimension.
356 This position is in this universe's sorting of all dimensions.
358 Parameters
359 ----------
360 name : `str`
361 Name of the dimension.
363 Returns
364 -------
365 index : `int`
366 Sorting index for this dimension.
368 Notes
369 -----
370 The dimension sort order for a universe is consistent with the element
371 order (all dimensions are elements), and either can be used to sort
372 dimensions if used consistently. But there are also some contexts in
373 which contiguous dimension-only indices are necessary or at least
374 desirable.
375 """
376 return self._dimensionIndices[name]
378 def expandDimensionNameSet(self, names: Set[str]) -> None:
379 """Expand a set of dimension names in-place.
381 Includes recursive dependencies.
383 This is an advanced interface for cases where constructing a
384 `DimensionGraph` (which also expands required dependencies) is
385 impossible or undesirable.
387 Parameters
388 ----------
389 names : `set` [ `str` ]
390 A true `set` of dimension names, to be expanded in-place.
391 """
392 # Keep iterating until the set of names stops growing. This is not as
393 # efficient as it could be, but we work pretty hard cache
394 # DimensionGraph instances to keep actual construction rare, so that
395 # shouldn't matter.
396 oldSize = len(names)
397 while True:
398 # iterate over a temporary copy so we can modify the original
399 for name in tuple(names):
400 names.update(self._dimensions[name].required.names)
401 names.update(self._dimensions[name].implied.names)
402 if oldSize == len(names):
403 break
404 else:
405 oldSize = len(names)
407 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
408 """Construct graph from iterable.
410 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
411 of `Dimension` instances and string names thereof.
413 Constructing `DimensionGraph` directly from names or dimension
414 instances is slightly more efficient when it is known in advance that
415 the iterable is not heterogenous.
417 Parameters
418 ----------
419 iterable: iterable of `Dimension` or `str`
420 Dimensions that must be included in the returned graph (their
421 dependencies will be as well).
423 Returns
424 -------
425 graph : `DimensionGraph`
426 A `DimensionGraph` instance containing all given dimensions.
427 """
428 names = set()
429 for item in iterable:
430 try:
431 names.add(item.name) # type: ignore
432 except AttributeError:
433 names.add(item)
434 return DimensionGraph(universe=self, names=names)
436 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
437 """Return a sorted version of the given iterable of dimension elements.
439 The universe's sort order is topological (an element's dependencies
440 precede it), with an unspecified (but deterministic) approach to
441 breaking ties.
443 Parameters
444 ----------
445 elements : iterable of `DimensionElement`.
446 Elements to be sorted.
447 reverse : `bool`, optional
448 If `True`, sort in the opposite order.
450 Returns
451 -------
452 sorted : `list` of `DimensionElement`
453 A sorted list containing the same elements that were given.
454 """
455 s = set(elements)
456 result = [element for element in self._elements if element in s or element.name in s]
457 if reverse:
458 result.reverse()
459 # mypy thinks this can return DimensionElements even if all the user
460 # passed it was Dimensions; we know better.
461 return result # type: ignore
463 # TODO: Remove this method on DM-38687.
464 @deprecated(
465 "Deprecated in favor of configurable dimension packers. Will be removed after v27.",
466 version="v26",
467 category=FutureWarning,
468 )
469 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
470 """Make a dimension packer.
472 Constructs a `DimensionPacker` that can pack data ID dictionaries
473 into unique integers.
475 Parameters
476 ----------
477 name : `str`
478 Name of the packer, matching a key in the "packers" section of the
479 dimension configuration.
480 dataId : `DataCoordinate`
481 Fully-expanded data ID that identifies the at least the "fixed"
482 dimensions of the packer (i.e. those that are assumed/given,
483 setting the space over which packed integer IDs are unique).
484 ``dataId.hasRecords()`` must return `True`.
485 """
486 return self._packers[name](self, dataId)
488 def getEncodeLength(self) -> int:
489 """Return encoded size of graph.
491 Returns the size (in bytes) of the encoded size of `DimensionGraph`
492 instances in this universe.
494 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
495 information.
496 """
497 return math.ceil(len(self._dimensions) / 8)
499 @classmethod
500 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse:
501 """Return an unpickled dimension universe.
503 Callable used for unpickling.
505 For internal use only.
506 """
507 if namespace is None:
508 # Old pickled universe.
509 namespace = _DEFAULT_NAMESPACE
510 try:
511 return cls._instances[version, namespace]
512 except KeyError as err:
513 raise pickle.UnpicklingError(
514 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
515 "not found. Note that DimensionUniverse objects are not "
516 "truly serialized; when using pickle to transfer them "
517 "between processes, an equivalent instance with the same "
518 "version must already exist in the receiving process."
519 ) from err
521 def __reduce__(self) -> tuple:
522 return (self._unpickle, (self._version, self._namespace))
524 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
525 # DimensionUniverse is recursively immutable; see note in @immutable
526 # decorator.
527 return self
529 # Class attributes below are shadowed by instance attributes, and are
530 # present just to hold the docstrings for those instance attributes.
532 empty: DimensionGraph
533 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
534 """
536 commonSkyPix: SkyPixDimension
537 """The special skypix dimension that is used to relate all other spatial
538 dimensions in the `Registry` database (`SkyPixDimension`).
539 """
541 dimensionConfig: DimensionConfig
542 """The configuration used to create this Universe (`DimensionConfig`)."""
544 _cache: Dict[FrozenSet[str], DimensionGraph]
546 _dimensions: NamedValueAbstractSet[Dimension]
548 _elements: NamedValueAbstractSet[DimensionElement]
550 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
552 _dimensionIndices: Dict[str, int]
554 _elementIndices: Dict[str, int]
556 _packers: Dict[str, DimensionPackerFactory]
558 _version: int
560 _namespace: str