Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 40%
165 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 15:14 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 15:14 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import logging
27import math
28import pickle
29from collections import defaultdict
30from collections.abc import Iterable, Mapping
31from typing import TYPE_CHECKING, Any, ClassVar, TypeVar
33from deprecated.sphinx import deprecated
34from lsst.utils.classes import cached_getter, immutable
36from .._topology import TopologicalFamily, TopologicalSpace
37from ..config import Config
38from ..named import NamedValueAbstractSet, NamedValueSet
39from ._config import _DEFAULT_NAMESPACE, DimensionConfig
40from ._database import DatabaseDimensionElement
41from ._elements import Dimension, DimensionElement
42from ._governor import GovernorDimension
43from ._graph import DimensionGraph
44from ._skypix import SkyPixDimension, SkyPixSystem
46if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
47 from ._coordinate import DataCoordinate
48 from ._packer import DimensionPacker, DimensionPackerFactory
49 from .construction import DimensionConstructionBuilder
52E = TypeVar("E", bound=DimensionElement)
53_LOG = logging.getLogger(__name__)
56@immutable
57class DimensionUniverse:
58 """Self-consistent set of dimensions.
60 A parent class that represents a complete, self-consistent set of
61 dimensions and their relationships.
63 `DimensionUniverse` is not a class-level singleton, but all instances are
64 tracked in a singleton map keyed by the version number and namespace
65 in the configuration they were loaded from. Because these universes
66 are solely responsible for constructing `DimensionElement` instances,
67 these are also indirectly tracked by that singleton as well.
69 Parameters
70 ----------
71 config : `Config`, optional
72 Configuration object from which dimension definitions can be extracted.
73 Ignored if ``builder`` is provided, or if ``version`` is provided and
74 an instance with that version already exists.
75 version : `int`, optional
76 Integer version for this `DimensionUniverse`. If not provided, a
77 version will be obtained from ``builder`` or ``config``.
78 namespace : `str`, optional
79 Namespace of this `DimensionUniverse`, combined with the version
80 to provide universe safety for registries that use different
81 dimension definitions.
82 builder : `DimensionConstructionBuilder`, optional
83 Builder object used to initialize a new instance. Ignored if
84 ``version`` is provided and an instance with that version already
85 exists. Should not have had `~DimensionConstructionBuilder.finish`
86 called; this will be called if needed by `DimensionUniverse`.
87 """
89 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {}
90 """Singleton dictionary of all instances, keyed by version.
92 For internal use only.
93 """
95 def __new__(
96 cls,
97 config: Config | None = None,
98 *,
99 version: int | None = None,
100 namespace: str | None = None,
101 builder: DimensionConstructionBuilder | None = None,
102 ) -> DimensionUniverse:
103 # Try to get a version first, to look for existing instances; try to
104 # do as little work as possible at this stage.
105 if version is None:
106 if builder is None:
107 config = DimensionConfig(config)
108 version = config["version"]
109 else:
110 version = builder.version
112 # Then a namespace.
113 if namespace is None:
114 if builder is None:
115 config = DimensionConfig(config)
116 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
117 else:
118 namespace = builder.namespace
119 # if still None use the default
120 if namespace is None:
121 namespace = _DEFAULT_NAMESPACE
123 # See if an equivalent instance already exists.
124 self: DimensionUniverse | None = cls._instances.get((version, namespace))
125 if self is not None:
126 return self
128 # Ensure we have a builder, building one from config if necessary.
129 if builder is None:
130 config = DimensionConfig(config)
131 builder = config.makeBuilder()
133 # Delegate to the builder for most of the construction work.
134 builder.finish()
136 # Create the universe instance and create core attributes, mostly
137 # copying from builder.
138 self = object.__new__(cls)
139 assert self is not None
140 self._cache = {}
141 self._dimensions = builder.dimensions
142 self._elements = builder.elements
143 self._topology = builder.topology
144 self._packers = builder.packers
145 self.dimensionConfig = builder.config
146 commonSkyPix = self._dimensions[builder.commonSkyPixName]
147 assert isinstance(commonSkyPix, SkyPixDimension)
148 self.commonSkyPix = commonSkyPix
150 # Attach self to all elements.
151 for element in self._elements:
152 element.universe = self
154 # Add attribute for special subsets of the graph.
155 self.empty = DimensionGraph(self, (), conform=False)
157 # Use the version number and namespace from the config as a key in
158 # the singleton dict containing all instances; that will let us
159 # transfer dimension objects between processes using pickle without
160 # actually going through real initialization, as long as a universe
161 # with the same version and namespace has already been constructed in
162 # the receiving process.
163 self._version = version
164 self._namespace = namespace
165 cls._instances[self._version, self._namespace] = self
167 # Build mappings from element to index. These are used for
168 # topological-sort comparison operators in DimensionElement itself.
169 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
170 # Same for dimension to index, sorted topologically across required
171 # and implied. This is used for encode/decode.
172 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
174 self._populates = defaultdict(NamedValueSet)
175 for element in self._elements:
176 if element.populated_by is not None:
177 self._populates[element.populated_by.name].add(element)
179 return self
181 @property
182 def version(self) -> int:
183 """The version number of this universe.
185 Returns
186 -------
187 version : `int`
188 An integer representing the version number of this universe.
189 Uniquely defined when combined with the `namespace`.
190 """
191 return self._version
193 @property
194 def namespace(self) -> str:
195 """The namespace associated with this universe.
197 Returns
198 -------
199 namespace : `str`
200 The namespace. When combined with the `version` can uniquely
201 define this universe.
202 """
203 return self._namespace
205 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
206 """Check compatibility between this `DimensionUniverse` and another.
208 Parameters
209 ----------
210 other : `DimensionUniverse`
211 The other `DimensionUniverse` to check for compatibility
213 Returns
214 -------
215 results : `bool`
216 If the other `DimensionUniverse` is compatible with this one return
217 `True`, else `False`
218 """
219 # Different namespaces mean that these universes cannot be compatible.
220 if self.namespace != other.namespace:
221 return False
222 if self.version != other.version:
223 _LOG.info(
224 "Universes share a namespace %r but have differing versions (%d != %d). "
225 " This could be okay but may be responsible for dimension errors later.",
226 self.namespace,
227 self.version,
228 other.version,
229 )
231 # For now assume compatibility if versions differ.
232 return True
234 def __repr__(self) -> str:
235 return f"DimensionUniverse({self._version}, {self._namespace})"
237 def __getitem__(self, name: str) -> DimensionElement:
238 return self._elements[name]
240 def __contains__(self, name: Any) -> bool:
241 return name in self._elements
243 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None:
244 """Return the `DimensionElement` with the given name or a default.
246 Parameters
247 ----------
248 name : `str`
249 Name of the element.
250 default : `DimensionElement`, optional
251 Element to return if the named one does not exist. Defaults to
252 `None`.
254 Returns
255 -------
256 element : `DimensionElement`
257 The named element.
258 """
259 return self._elements.get(name, default)
261 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
262 """Return a set of all static elements in this universe.
264 Non-static elements that are created as needed may also exist, but
265 these are guaranteed to have no direct relationships to other elements
266 (though they may have spatial or temporal relationships).
268 Returns
269 -------
270 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
271 A frozen set of `DimensionElement` instances.
272 """
273 return self._elements
275 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
276 """Return a set of all static dimensions in this universe.
278 Non-static dimensions that are created as needed may also exist, but
279 these are guaranteed to have no direct relationships to other elements
280 (though they may have spatial or temporal relationships).
282 Returns
283 -------
284 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
285 A frozen set of `Dimension` instances.
286 """
287 return self._dimensions
289 @cached_getter
290 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
291 """Return a set of all `GovernorDimension` instances in this universe.
293 Returns
294 -------
295 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
296 A frozen set of `GovernorDimension` instances.
297 """
298 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
300 @cached_getter
301 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
302 """Return set of all `DatabaseDimensionElement` instances in universe.
304 This does not include `GovernorDimension` instances, which are backed
305 by the database but do not inherit from `DatabaseDimensionElement`.
307 Returns
308 -------
309 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
310 A frozen set of `DatabaseDimensionElement` instances.
311 """
312 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
314 @property
315 @cached_getter
316 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
317 """All skypix systems known to this universe.
319 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
320 """
321 return NamedValueSet(
322 [
323 family
324 for family in self._topology[TopologicalSpace.SPATIAL]
325 if isinstance(family, SkyPixSystem)
326 ]
327 ).freeze()
329 def getElementIndex(self, name: str) -> int:
330 """Return the position of the named dimension element.
332 The position is in this universe's sorting of all elements.
334 Parameters
335 ----------
336 name : `str`
337 Name of the element.
339 Returns
340 -------
341 index : `int`
342 Sorting index for this element.
343 """
344 return self._elementIndices[name]
346 def getDimensionIndex(self, name: str) -> int:
347 """Return the position of the named dimension.
349 This position is in this universe's sorting of all dimensions.
351 Parameters
352 ----------
353 name : `str`
354 Name of the dimension.
356 Returns
357 -------
358 index : `int`
359 Sorting index for this dimension.
361 Notes
362 -----
363 The dimension sort order for a universe is consistent with the element
364 order (all dimensions are elements), and either can be used to sort
365 dimensions if used consistently. But there are also some contexts in
366 which contiguous dimension-only indices are necessary or at least
367 desirable.
368 """
369 return self._dimensionIndices[name]
371 def expandDimensionNameSet(self, names: set[str]) -> None:
372 """Expand a set of dimension names in-place.
374 Includes recursive dependencies.
376 This is an advanced interface for cases where constructing a
377 `DimensionGraph` (which also expands required dependencies) is
378 impossible or undesirable.
380 Parameters
381 ----------
382 names : `set` [ `str` ]
383 A true `set` of dimension names, to be expanded in-place.
384 """
385 # Keep iterating until the set of names stops growing. This is not as
386 # efficient as it could be, but we work pretty hard cache
387 # DimensionGraph instances to keep actual construction rare, so that
388 # shouldn't matter.
389 oldSize = len(names)
390 while True:
391 # iterate over a temporary copy so we can modify the original
392 for name in tuple(names):
393 names.update(self._dimensions[name].required.names)
394 names.update(self._dimensions[name].implied.names)
395 if oldSize == len(names):
396 break
397 else:
398 oldSize = len(names)
400 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph:
401 """Construct graph from iterable.
403 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
404 of `Dimension` instances and string names thereof.
406 Constructing `DimensionGraph` directly from names or dimension
407 instances is slightly more efficient when it is known in advance that
408 the iterable is not heterogenous.
410 Parameters
411 ----------
412 iterable: iterable of `Dimension` or `str`
413 Dimensions that must be included in the returned graph (their
414 dependencies will be as well).
416 Returns
417 -------
418 graph : `DimensionGraph`
419 A `DimensionGraph` instance containing all given dimensions.
420 """
421 names = set()
422 for item in iterable:
423 try:
424 names.add(item.name) # type: ignore
425 except AttributeError:
426 names.add(item)
427 return DimensionGraph(universe=self, names=names)
429 def sorted(self, elements: Iterable[E | str], *, reverse: bool = False) -> list[E]:
430 """Return a sorted version of the given iterable of dimension elements.
432 The universe's sort order is topological (an element's dependencies
433 precede it), with an unspecified (but deterministic) approach to
434 breaking ties.
436 Parameters
437 ----------
438 elements : iterable of `DimensionElement`.
439 Elements to be sorted.
440 reverse : `bool`, optional
441 If `True`, sort in the opposite order.
443 Returns
444 -------
445 sorted : `list` of `DimensionElement`
446 A sorted list containing the same elements that were given.
447 """
448 s = set(elements)
449 result = [element for element in self._elements if element in s or element.name in s]
450 if reverse:
451 result.reverse()
452 # mypy thinks this can return DimensionElements even if all the user
453 # passed it was Dimensions; we know better.
454 return result # type: ignore
456 # TODO: Remove this method on DM-38687.
457 @deprecated(
458 "Deprecated in favor of configurable dimension packers. Will be removed after v26.",
459 version="v26",
460 category=FutureWarning,
461 )
462 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
463 """Make a dimension packer.
465 Constructs a `DimensionPacker` that can pack data ID dictionaries
466 into unique integers.
468 Parameters
469 ----------
470 name : `str`
471 Name of the packer, matching a key in the "packers" section of the
472 dimension configuration.
473 dataId : `DataCoordinate`
474 Fully-expanded data ID that identifies the at least the "fixed"
475 dimensions of the packer (i.e. those that are assumed/given,
476 setting the space over which packed integer IDs are unique).
477 ``dataId.hasRecords()`` must return `True`.
478 """
479 return self._packers[name](self, dataId)
481 def getEncodeLength(self) -> int:
482 """Return encoded size of graph.
484 Returns the size (in bytes) of the encoded size of `DimensionGraph`
485 instances in this universe.
487 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
488 information.
489 """
490 return math.ceil(len(self._dimensions) / 8)
492 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]:
493 """Return the set of `DimensionElement` objects whose
494 `~DimensionElement.populated_by` atttribute is the given dimension.
495 """
496 return self._populates[dimension.name]
498 @classmethod
499 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse:
500 """Return an unpickled dimension universe.
502 Callable used for unpickling.
504 For internal use only.
505 """
506 if namespace is None:
507 # Old pickled universe.
508 namespace = _DEFAULT_NAMESPACE
509 try:
510 return cls._instances[version, namespace]
511 except KeyError as err:
512 raise pickle.UnpicklingError(
513 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
514 "not found. Note that DimensionUniverse objects are not "
515 "truly serialized; when using pickle to transfer them "
516 "between processes, an equivalent instance with the same "
517 "version must already exist in the receiving process."
518 ) from err
520 def __reduce__(self) -> tuple:
521 return (self._unpickle, (self._version, self._namespace))
523 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
524 # DimensionUniverse is recursively immutable; see note in @immutable
525 # decorator.
526 return self
528 # Class attributes below are shadowed by instance attributes, and are
529 # present just to hold the docstrings for those instance attributes.
531 empty: DimensionGraph
532 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
533 """
535 commonSkyPix: SkyPixDimension
536 """The special skypix dimension that is used to relate all other spatial
537 dimensions in the `Registry` database (`SkyPixDimension`).
538 """
540 dimensionConfig: DimensionConfig
541 """The configuration used to create this Universe (`DimensionConfig`)."""
543 _cache: dict[frozenset[str], DimensionGraph]
545 _dimensions: NamedValueAbstractSet[Dimension]
547 _elements: NamedValueAbstractSet[DimensionElement]
549 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
551 _dimensionIndices: dict[str, int]
553 _elementIndices: dict[str, int]
555 _packers: dict[str, DimensionPackerFactory]
557 _populates: defaultdict[str, NamedValueSet[DimensionElement]]
559 _version: int
561 _namespace: str