Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 36%
157 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import logging
27import math
28import pickle
29from collections.abc import Iterable, Mapping
30from typing import TYPE_CHECKING, Any, ClassVar, TypeVar
32from deprecated.sphinx import deprecated
33from lsst.utils.classes import cached_getter, immutable
35from .._topology import TopologicalFamily, TopologicalSpace
36from ..config import Config
37from ..named import NamedValueAbstractSet, NamedValueSet
38from ._config import _DEFAULT_NAMESPACE, DimensionConfig
39from ._database import DatabaseDimensionElement
40from ._elements import Dimension, DimensionElement
41from ._governor import GovernorDimension
42from ._graph import DimensionGraph
43from ._skypix import SkyPixDimension, SkyPixSystem
45if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
46 from ._coordinate import DataCoordinate
47 from ._packer import DimensionPacker, DimensionPackerFactory
48 from .construction import DimensionConstructionBuilder
51E = TypeVar("E", bound=DimensionElement)
52_LOG = logging.getLogger(__name__)
55@immutable
56class DimensionUniverse:
57 """Self-consistent set of dimensions.
59 A parent class that represents a complete, self-consistent set of
60 dimensions and their relationships.
62 `DimensionUniverse` is not a class-level singleton, but all instances are
63 tracked in a singleton map keyed by the version number and namespace
64 in the configuration they were loaded from. Because these universes
65 are solely responsible for constructing `DimensionElement` instances,
66 these are also indirectly tracked by that singleton as well.
68 Parameters
69 ----------
70 config : `Config`, optional
71 Configuration object from which dimension definitions can be extracted.
72 Ignored if ``builder`` is provided, or if ``version`` is provided and
73 an instance with that version already exists.
74 version : `int`, optional
75 Integer version for this `DimensionUniverse`. If not provided, a
76 version will be obtained from ``builder`` or ``config``.
77 namespace : `str`, optional
78 Namespace of this `DimensionUniverse`, combined with the version
79 to provide universe safety for registries that use different
80 dimension definitions.
81 builder : `DimensionConstructionBuilder`, optional
82 Builder object used to initialize a new instance. Ignored if
83 ``version`` is provided and an instance with that version already
84 exists. Should not have had `~DimensionConstructionBuilder.finish`
85 called; this will be called if needed by `DimensionUniverse`.
86 """
88 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {}
89 """Singleton dictionary of all instances, keyed by version.
91 For internal use only.
92 """
94 def __new__(
95 cls,
96 config: Config | None = None,
97 *,
98 version: int | None = None,
99 namespace: str | None = None,
100 builder: DimensionConstructionBuilder | None = None,
101 ) -> DimensionUniverse:
102 # Try to get a version first, to look for existing instances; try to
103 # do as little work as possible at this stage.
104 if version is None:
105 if builder is None:
106 config = DimensionConfig(config)
107 version = config["version"]
108 else:
109 version = builder.version
111 # Then a namespace.
112 if namespace is None:
113 if builder is None:
114 config = DimensionConfig(config)
115 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
116 else:
117 namespace = builder.namespace
118 # if still None use the default
119 if namespace is None:
120 namespace = _DEFAULT_NAMESPACE
122 # See if an equivalent instance already exists.
123 self: DimensionUniverse | None = cls._instances.get((version, namespace))
124 if self is not None:
125 return self
127 # Ensure we have a builder, building one from config if necessary.
128 if builder is None:
129 config = DimensionConfig(config)
130 builder = config.makeBuilder()
132 # Delegate to the builder for most of the construction work.
133 builder.finish()
135 # Create the universe instance and create core attributes, mostly
136 # copying from builder.
137 self = object.__new__(cls)
138 assert self is not None
139 self._cache = {}
140 self._dimensions = builder.dimensions
141 self._elements = builder.elements
142 self._topology = builder.topology
143 self._packers = builder.packers
144 self.dimensionConfig = builder.config
145 commonSkyPix = self._dimensions[builder.commonSkyPixName]
146 assert isinstance(commonSkyPix, SkyPixDimension)
147 self.commonSkyPix = commonSkyPix
149 # Attach self to all elements.
150 for element in self._elements:
151 element.universe = self
153 # Add attribute for special subsets of the graph.
154 self.empty = DimensionGraph(self, (), conform=False)
156 # Use the version number and namespace from the config as a key in
157 # the singleton dict containing all instances; that will let us
158 # transfer dimension objects between processes using pickle without
159 # actually going through real initialization, as long as a universe
160 # with the same version and namespace has already been constructed in
161 # the receiving process.
162 self._version = version
163 self._namespace = namespace
164 cls._instances[self._version, self._namespace] = self
166 # Build mappings from element to index. These are used for
167 # topological-sort comparison operators in DimensionElement itself.
168 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
169 # Same for dimension to index, sorted topologically across required
170 # and implied. This is used for encode/decode.
171 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
173 return self
175 @property
176 def version(self) -> int:
177 """The version number of this universe.
179 Returns
180 -------
181 version : `int`
182 An integer representing the version number of this universe.
183 Uniquely defined when combined with the `namespace`.
184 """
185 return self._version
187 @property
188 def namespace(self) -> str:
189 """The namespace associated with this universe.
191 Returns
192 -------
193 namespace : `str`
194 The namespace. When combined with the `version` can uniquely
195 define this universe.
196 """
197 return self._namespace
199 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
200 """Check compatibility between this `DimensionUniverse` and another.
202 Parameters
203 ----------
204 other : `DimensionUniverse`
205 The other `DimensionUniverse` to check for compatibility
207 Returns
208 -------
209 results : `bool`
210 If the other `DimensionUniverse` is compatible with this one return
211 `True`, else `False`
212 """
213 # Different namespaces mean that these universes cannot be compatible.
214 if self.namespace != other.namespace:
215 return False
216 if self.version != other.version:
217 _LOG.info(
218 "Universes share a namespace %r but have differing versions (%d != %d). "
219 " This could be okay but may be responsible for dimension errors later.",
220 self.namespace,
221 self.version,
222 other.version,
223 )
225 # For now assume compatibility if versions differ.
226 return True
228 def __repr__(self) -> str:
229 return f"DimensionUniverse({self._version}, {self._namespace})"
231 def __getitem__(self, name: str) -> DimensionElement:
232 return self._elements[name]
234 def __contains__(self, name: Any) -> bool:
235 return name in self._elements
237 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None:
238 """Return the `DimensionElement` with the given name or a default.
240 Parameters
241 ----------
242 name : `str`
243 Name of the element.
244 default : `DimensionElement`, optional
245 Element to return if the named one does not exist. Defaults to
246 `None`.
248 Returns
249 -------
250 element : `DimensionElement`
251 The named element.
252 """
253 return self._elements.get(name, default)
255 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
256 """Return a set of all static elements in this universe.
258 Non-static elements that are created as needed may also exist, but
259 these are guaranteed to have no direct relationships to other elements
260 (though they may have spatial or temporal relationships).
262 Returns
263 -------
264 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
265 A frozen set of `DimensionElement` instances.
266 """
267 return self._elements
269 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
270 """Return a set of all static dimensions in this universe.
272 Non-static dimensions that are created as needed may also exist, but
273 these are guaranteed to have no direct relationships to other elements
274 (though they may have spatial or temporal relationships).
276 Returns
277 -------
278 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
279 A frozen set of `Dimension` instances.
280 """
281 return self._dimensions
283 @cached_getter
284 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
285 """Return a set of all `GovernorDimension` instances in this universe.
287 Returns
288 -------
289 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
290 A frozen set of `GovernorDimension` instances.
291 """
292 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
294 @cached_getter
295 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
296 """Return set of all `DatabaseDimensionElement` instances in universe.
298 This does not include `GovernorDimension` instances, which are backed
299 by the database but do not inherit from `DatabaseDimensionElement`.
301 Returns
302 -------
303 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
304 A frozen set of `DatabaseDimensionElement` instances.
305 """
306 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
308 @property
309 @cached_getter
310 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
311 """All skypix systems known to this universe.
313 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
314 """
315 return NamedValueSet(
316 [
317 family
318 for family in self._topology[TopologicalSpace.SPATIAL]
319 if isinstance(family, SkyPixSystem)
320 ]
321 ).freeze()
323 def getElementIndex(self, name: str) -> int:
324 """Return the position of the named dimension element.
326 The position is in this universe's sorting of all elements.
328 Parameters
329 ----------
330 name : `str`
331 Name of the element.
333 Returns
334 -------
335 index : `int`
336 Sorting index for this element.
337 """
338 return self._elementIndices[name]
340 def getDimensionIndex(self, name: str) -> int:
341 """Return the position of the named dimension.
343 This position is in this universe's sorting of all dimensions.
345 Parameters
346 ----------
347 name : `str`
348 Name of the dimension.
350 Returns
351 -------
352 index : `int`
353 Sorting index for this dimension.
355 Notes
356 -----
357 The dimension sort order for a universe is consistent with the element
358 order (all dimensions are elements), and either can be used to sort
359 dimensions if used consistently. But there are also some contexts in
360 which contiguous dimension-only indices are necessary or at least
361 desirable.
362 """
363 return self._dimensionIndices[name]
365 def expandDimensionNameSet(self, names: set[str]) -> None:
366 """Expand a set of dimension names in-place.
368 Includes recursive dependencies.
370 This is an advanced interface for cases where constructing a
371 `DimensionGraph` (which also expands required dependencies) is
372 impossible or undesirable.
374 Parameters
375 ----------
376 names : `set` [ `str` ]
377 A true `set` of dimension names, to be expanded in-place.
378 """
379 # Keep iterating until the set of names stops growing. This is not as
380 # efficient as it could be, but we work pretty hard cache
381 # DimensionGraph instances to keep actual construction rare, so that
382 # shouldn't matter.
383 oldSize = len(names)
384 while True:
385 # iterate over a temporary copy so we can modify the original
386 for name in tuple(names):
387 names.update(self._dimensions[name].required.names)
388 names.update(self._dimensions[name].implied.names)
389 if oldSize == len(names):
390 break
391 else:
392 oldSize = len(names)
394 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph:
395 """Construct graph from iterable.
397 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
398 of `Dimension` instances and string names thereof.
400 Constructing `DimensionGraph` directly from names or dimension
401 instances is slightly more efficient when it is known in advance that
402 the iterable is not heterogenous.
404 Parameters
405 ----------
406 iterable: iterable of `Dimension` or `str`
407 Dimensions that must be included in the returned graph (their
408 dependencies will be as well).
410 Returns
411 -------
412 graph : `DimensionGraph`
413 A `DimensionGraph` instance containing all given dimensions.
414 """
415 names = set()
416 for item in iterable:
417 try:
418 names.add(item.name) # type: ignore
419 except AttributeError:
420 names.add(item)
421 return DimensionGraph(universe=self, names=names)
423 def sorted(self, elements: Iterable[E | str], *, reverse: bool = False) -> list[E]:
424 """Return a sorted version of the given iterable of dimension elements.
426 The universe's sort order is topological (an element's dependencies
427 precede it), with an unspecified (but deterministic) approach to
428 breaking ties.
430 Parameters
431 ----------
432 elements : iterable of `DimensionElement`.
433 Elements to be sorted.
434 reverse : `bool`, optional
435 If `True`, sort in the opposite order.
437 Returns
438 -------
439 sorted : `list` of `DimensionElement`
440 A sorted list containing the same elements that were given.
441 """
442 s = set(elements)
443 result = [element for element in self._elements if element in s or element.name in s]
444 if reverse:
445 result.reverse()
446 # mypy thinks this can return DimensionElements even if all the user
447 # passed it was Dimensions; we know better.
448 return result # type: ignore
450 # TODO: Remove this method on DM-38687.
451 @deprecated(
452 "Deprecated in favor of configurable dimension packers. Will be removed after v27.",
453 version="v26",
454 category=FutureWarning,
455 )
456 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
457 """Make a dimension packer.
459 Constructs a `DimensionPacker` that can pack data ID dictionaries
460 into unique integers.
462 Parameters
463 ----------
464 name : `str`
465 Name of the packer, matching a key in the "packers" section of the
466 dimension configuration.
467 dataId : `DataCoordinate`
468 Fully-expanded data ID that identifies the at least the "fixed"
469 dimensions of the packer (i.e. those that are assumed/given,
470 setting the space over which packed integer IDs are unique).
471 ``dataId.hasRecords()`` must return `True`.
472 """
473 return self._packers[name](self, dataId)
475 def getEncodeLength(self) -> int:
476 """Return encoded size of graph.
478 Returns the size (in bytes) of the encoded size of `DimensionGraph`
479 instances in this universe.
481 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
482 information.
483 """
484 return math.ceil(len(self._dimensions) / 8)
486 @classmethod
487 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse:
488 """Return an unpickled dimension universe.
490 Callable used for unpickling.
492 For internal use only.
493 """
494 if namespace is None:
495 # Old pickled universe.
496 namespace = _DEFAULT_NAMESPACE
497 try:
498 return cls._instances[version, namespace]
499 except KeyError as err:
500 raise pickle.UnpicklingError(
501 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
502 "not found. Note that DimensionUniverse objects are not "
503 "truly serialized; when using pickle to transfer them "
504 "between processes, an equivalent instance with the same "
505 "version must already exist in the receiving process."
506 ) from err
508 def __reduce__(self) -> tuple:
509 return (self._unpickle, (self._version, self._namespace))
511 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
512 # DimensionUniverse is recursively immutable; see note in @immutable
513 # decorator.
514 return self
516 # Class attributes below are shadowed by instance attributes, and are
517 # present just to hold the docstrings for those instance attributes.
519 empty: DimensionGraph
520 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
521 """
523 commonSkyPix: SkyPixDimension
524 """The special skypix dimension that is used to relate all other spatial
525 dimensions in the `Registry` database (`SkyPixDimension`).
526 """
528 dimensionConfig: DimensionConfig
529 """The configuration used to create this Universe (`DimensionConfig`)."""
531 _cache: dict[frozenset[str], DimensionGraph]
533 _dimensions: NamedValueAbstractSet[Dimension]
535 _elements: NamedValueAbstractSet[DimensionElement]
537 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
539 _dimensionIndices: dict[str, int]
541 _elementIndices: dict[str, int]
543 _packers: dict[str, DimensionPackerFactory]
545 _version: int
547 _namespace: str