Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 33%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 TYPE_CHECKING,
30 ClassVar,
31 Dict,
32 FrozenSet,
33 Iterable,
34 List,
35 Mapping,
36 Optional,
37 Set,
38 Tuple,
39 TypeVar,
40 Union,
41)
43from lsst.utils.classes import cached_getter, immutable
45from .._topology import TopologicalFamily, TopologicalSpace
46from ..config import Config
47from ..named import NamedValueAbstractSet, NamedValueSet
48from ._config import _DEFAULT_NAMESPACE, DimensionConfig
49from ._database import DatabaseDimensionElement
50from ._elements import Dimension, DimensionElement
51from ._governor import GovernorDimension
52from ._graph import DimensionGraph
53from ._skypix import SkyPixDimension, SkyPixSystem
55if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 55 ↛ 56line 55 didn't jump to line 56, because the condition on line 55 was never true
56 from ._coordinate import DataCoordinate
57 from ._packer import DimensionPacker, DimensionPackerFactory
58 from .construction import DimensionConstructionBuilder
61E = TypeVar("E", bound=DimensionElement)
64@immutable
65class DimensionUniverse:
66 """Self-consistent set of dimensions.
68 A parent class that represents a complete, self-consistent set of
69 dimensions and their relationships.
71 `DimensionUniverse` is not a class-level singleton, but all instances are
72 tracked in a singleton map keyed by the version number and namespace
73 in the configuration they were loaded from. Because these universes
74 are solely responsible for constructing `DimensionElement` instances,
75 these are also indirectly tracked by that singleton as well.
77 Parameters
78 ----------
79 config : `Config`, optional
80 Configuration object from which dimension definitions can be extracted.
81 Ignored if ``builder`` is provided, or if ``version`` is provided and
82 an instance with that version already exists.
83 version : `int`, optional
84 Integer version for this `DimensionUniverse`. If not provided, a
85 version will be obtained from ``builder`` or ``config``.
86 namespace : `str`, optional
87 Namespace of this `DimensionUniverse`, combined with the version
88 to provide universe safety for registries that use different
89 dimension definitions.
90 builder : `DimensionConstructionBuilder`, optional
91 Builder object used to initialize a new instance. Ignored if
92 ``version`` is provided and an instance with that version already
93 exists. Should not have had `~DimensionConstructionBuilder.finish`
94 called; this will be called if needed by `DimensionUniverse`.
95 """
97 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {}
98 """Singleton dictionary of all instances, keyed by version.
100 For internal use only.
101 """
103 def __new__(
104 cls,
105 config: Optional[Config] = None,
106 *,
107 version: Optional[int] = None,
108 namespace: Optional[str] = None,
109 builder: Optional[DimensionConstructionBuilder] = None,
110 ) -> DimensionUniverse:
111 # Try to get a version first, to look for existing instances; try to
112 # do as little work as possible at this stage.
113 if version is None:
114 if builder is None:
115 config = DimensionConfig(config)
116 version = config["version"]
117 else:
118 version = builder.version
120 # Then a namespace.
121 if namespace is None:
122 if builder is None:
123 config = DimensionConfig(config)
124 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
125 else:
126 namespace = builder.namespace
127 # if still None use the default
128 if namespace is None:
129 namespace = _DEFAULT_NAMESPACE
131 # See if an equivalent instance already exists.
132 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace))
133 if self is not None:
134 return self
136 # Ensure we have a builder, building one from config if necessary.
137 if builder is None:
138 config = DimensionConfig(config)
139 builder = config.makeBuilder()
141 # Delegate to the builder for most of the construction work.
142 builder.finish()
144 # Create the universe instance and create core attributes, mostly
145 # copying from builder.
146 self = object.__new__(cls)
147 assert self is not None
148 self._cache = {}
149 self._dimensions = builder.dimensions
150 self._elements = builder.elements
151 self._topology = builder.topology
152 self._packers = builder.packers
153 self.dimensionConfig = builder.config
154 commonSkyPix = self._dimensions[builder.commonSkyPixName]
155 assert isinstance(commonSkyPix, SkyPixDimension)
156 self.commonSkyPix = commonSkyPix
158 # Attach self to all elements.
159 for element in self._elements:
160 element.universe = self
162 # Add attribute for special subsets of the graph.
163 self.empty = DimensionGraph(self, (), conform=False)
165 # Use the version number and namespace from the config as a key in
166 # the singleton dict containing all instances; that will let us
167 # transfer dimension objects between processes using pickle without
168 # actually going through real initialization, as long as a universe
169 # with the same version and namespace has already been constructed in
170 # the receiving process.
171 self._version = version
172 self._namespace = namespace
173 cls._instances[self._version, self._namespace] = self
175 # Build mappings from element to index. These are used for
176 # topological-sort comparison operators in DimensionElement itself.
177 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
178 # Same for dimension to index, sorted topologically across required
179 # and implied. This is used for encode/decode.
180 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
182 return self
184 def __repr__(self) -> str:
185 return f"DimensionUniverse({self._version}, {self._namespace})"
187 def __getitem__(self, name: str) -> DimensionElement:
188 return self._elements[name]
190 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
191 """Return the `DimensionElement` with the given name or a default.
193 Parameters
194 ----------
195 name : `str`
196 Name of the element.
197 default : `DimensionElement`, optional
198 Element to return if the named one does not exist. Defaults to
199 `None`.
201 Returns
202 -------
203 element : `DimensionElement`
204 The named element.
205 """
206 return self._elements.get(name, default)
208 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
209 """Return a set of all static elements in this universe.
211 Non-static elements that are created as needed may also exist, but
212 these are guaranteed to have no direct relationships to other elements
213 (though they may have spatial or temporal relationships).
215 Returns
216 -------
217 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
218 A frozen set of `DimensionElement` instances.
219 """
220 return self._elements
222 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
223 """Return a set of all static dimensions in this universe.
225 Non-static dimensions that are created as needed may also exist, but
226 these are guaranteed to have no direct relationships to other elements
227 (though they may have spatial or temporal relationships).
229 Returns
230 -------
231 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
232 A frozen set of `Dimension` instances.
233 """
234 return self._dimensions
236 @cached_getter
237 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
238 """Return a set of all `GovernorDimension` instances in this universe.
240 Returns
241 -------
242 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
243 A frozen set of `GovernorDimension` instances.
244 """
245 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
247 @cached_getter
248 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
249 """Return set of all `DatabaseDimensionElement` instances in universe.
251 This does not include `GovernorDimension` instances, which are backed
252 by the database but do not inherit from `DatabaseDimensionElement`.
254 Returns
255 -------
256 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
257 A frozen set of `DatabaseDimensionElement` instances.
258 """
259 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
261 @property # type: ignore
262 @cached_getter
263 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
264 """All skypix systems known to this universe.
266 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
267 """
268 return NamedValueSet(
269 [
270 family
271 for family in self._topology[TopologicalSpace.SPATIAL]
272 if isinstance(family, SkyPixSystem)
273 ]
274 ).freeze()
276 def getElementIndex(self, name: str) -> int:
277 """Return the position of the named dimension element.
279 The position is in this universe's sorting of all elements.
281 Parameters
282 ----------
283 name : `str`
284 Name of the element.
286 Returns
287 -------
288 index : `int`
289 Sorting index for this element.
290 """
291 return self._elementIndices[name]
293 def getDimensionIndex(self, name: str) -> int:
294 """Return the position of the named dimension.
296 This position is in this universe's sorting of all dimensions.
298 Parameters
299 ----------
300 name : `str`
301 Name of the dimension.
303 Returns
304 -------
305 index : `int`
306 Sorting index for this dimension.
308 Notes
309 -----
310 The dimension sort order for a universe is consistent with the element
311 order (all dimensions are elements), and either can be used to sort
312 dimensions if used consistently. But there are also some contexts in
313 which contiguous dimension-only indices are necessary or at least
314 desirable.
315 """
316 return self._dimensionIndices[name]
318 def expandDimensionNameSet(self, names: Set[str]) -> None:
319 """Expand a set of dimension names in-place.
321 Includes recursive dependencies.
323 This is an advanced interface for cases where constructing a
324 `DimensionGraph` (which also expands required dependencies) is
325 impossible or undesirable.
327 Parameters
328 ----------
329 names : `set` [ `str` ]
330 A true `set` of dimension names, to be expanded in-place.
331 """
332 # Keep iterating until the set of names stops growing. This is not as
333 # efficient as it could be, but we work pretty hard cache
334 # DimensionGraph instances to keep actual construction rare, so that
335 # shouldn't matter.
336 oldSize = len(names)
337 while True:
338 # iterate over a temporary copy so we can modify the original
339 for name in tuple(names):
340 names.update(self._dimensions[name].required.names)
341 names.update(self._dimensions[name].implied.names)
342 if oldSize == len(names):
343 break
344 else:
345 oldSize = len(names)
347 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
348 """Construct graph from iterable.
350 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
351 of `Dimension` instances and string names thereof.
353 Constructing `DimensionGraph` directly from names or dimension
354 instances is slightly more efficient when it is known in advance that
355 the iterable is not heterogenous.
357 Parameters
358 ----------
359 iterable: iterable of `Dimension` or `str`
360 Dimensions that must be included in the returned graph (their
361 dependencies will be as well).
363 Returns
364 -------
365 graph : `DimensionGraph`
366 A `DimensionGraph` instance containing all given dimensions.
367 """
368 names = set()
369 for item in iterable:
370 try:
371 names.add(item.name) # type: ignore
372 except AttributeError:
373 names.add(item)
374 return DimensionGraph(universe=self, names=names)
376 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
377 """Return a sorted version of the given iterable of dimension elements.
379 The universe's sort order is topological (an element's dependencies
380 precede it), with an unspecified (but deterministic) approach to
381 breaking ties.
383 Parameters
384 ----------
385 elements : iterable of `DimensionElement`.
386 Elements to be sorted.
387 reverse : `bool`, optional
388 If `True`, sort in the opposite order.
390 Returns
391 -------
392 sorted : `list` of `DimensionElement`
393 A sorted list containing the same elements that were given.
394 """
395 s = set(elements)
396 result = [element for element in self._elements if element in s or element.name in s]
397 if reverse:
398 result.reverse()
399 # mypy thinks this can return DimensionElements even if all the user
400 # passed it was Dimensions; we know better.
401 return result # type: ignore
403 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
404 """Make a dimension packer.
406 Constructs a `DimensionPacker` that can pack data ID dictionaries
407 into unique integers.
409 Parameters
410 ----------
411 name : `str`
412 Name of the packer, matching a key in the "packers" section of the
413 dimension configuration.
414 dataId : `DataCoordinate`
415 Fully-expanded data ID that identifies the at least the "fixed"
416 dimensions of the packer (i.e. those that are assumed/given,
417 setting the space over which packed integer IDs are unique).
418 ``dataId.hasRecords()`` must return `True`.
419 """
420 return self._packers[name](self, dataId)
422 def getEncodeLength(self) -> int:
423 """Return encoded size of graph.
425 Returns the size (in bytes) of the encoded size of `DimensionGraph`
426 instances in this universe.
428 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
429 information.
430 """
431 return math.ceil(len(self._dimensions) / 8)
433 @classmethod
434 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse:
435 """Return an unpickled dimension universe.
437 Callable used for unpickling.
439 For internal use only.
440 """
441 if namespace is None:
442 # Old pickled universe.
443 namespace = _DEFAULT_NAMESPACE
444 try:
445 return cls._instances[version, namespace]
446 except KeyError as err:
447 raise pickle.UnpicklingError(
448 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
449 f"not found. Note that DimensionUniverse objects are not "
450 f"truly serialized; when using pickle to transfer them "
451 f"between processes, an equivalent instance with the same "
452 f"version must already exist in the receiving process."
453 ) from err
455 def __reduce__(self) -> tuple:
456 return (self._unpickle, (self._version, self._namespace))
458 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
459 # DimensionUniverse is recursively immutable; see note in @immutable
460 # decorator.
461 return self
463 # Class attributes below are shadowed by instance attributes, and are
464 # present just to hold the docstrings for those instance attributes.
466 empty: DimensionGraph
467 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
468 """
470 commonSkyPix: SkyPixDimension
471 """The special skypix dimension that is used to relate all other spatial
472 dimensions in the `Registry` database (`SkyPixDimension`).
473 """
475 dimensionConfig: DimensionConfig
476 """The configuration used to create this Universe (`DimensionConfig`)."""
478 _cache: Dict[FrozenSet[str], DimensionGraph]
480 _dimensions: NamedValueAbstractSet[Dimension]
482 _elements: NamedValueAbstractSet[DimensionElement]
484 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
486 _dimensionIndices: Dict[str, int]
488 _elementIndices: Dict[str, int]
490 _packers: Dict[str, DimensionPackerFactory]
492 _version: int
494 _namespace: Optional[str]