Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 35%
150 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-15 02:06 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-15 02:06 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 TYPE_CHECKING,
30 Any,
31 ClassVar,
32 Dict,
33 FrozenSet,
34 Iterable,
35 List,
36 Mapping,
37 Optional,
38 Set,
39 Tuple,
40 TypeVar,
41 Union,
42)
44from lsst.utils.classes import cached_getter, immutable
46from .._topology import TopologicalFamily, TopologicalSpace
47from ..config import Config
48from ..named import NamedValueAbstractSet, NamedValueSet
49from ._config import _DEFAULT_NAMESPACE, DimensionConfig
50from ._database import DatabaseDimensionElement
51from ._elements import Dimension, DimensionElement
52from ._governor import GovernorDimension
53from ._graph import DimensionGraph
54from ._skypix import SkyPixDimension, SkyPixSystem
56if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 56 ↛ 57line 56 didn't jump to line 57, because the condition on line 56 was never true
57 from ._coordinate import DataCoordinate
58 from ._packer import DimensionPacker, DimensionPackerFactory
59 from .construction import DimensionConstructionBuilder
62E = TypeVar("E", bound=DimensionElement)
65@immutable
66class DimensionUniverse:
67 """Self-consistent set of dimensions.
69 A parent class that represents a complete, self-consistent set of
70 dimensions and their relationships.
72 `DimensionUniverse` is not a class-level singleton, but all instances are
73 tracked in a singleton map keyed by the version number and namespace
74 in the configuration they were loaded from. Because these universes
75 are solely responsible for constructing `DimensionElement` instances,
76 these are also indirectly tracked by that singleton as well.
78 Parameters
79 ----------
80 config : `Config`, optional
81 Configuration object from which dimension definitions can be extracted.
82 Ignored if ``builder`` is provided, or if ``version`` is provided and
83 an instance with that version already exists.
84 version : `int`, optional
85 Integer version for this `DimensionUniverse`. If not provided, a
86 version will be obtained from ``builder`` or ``config``.
87 namespace : `str`, optional
88 Namespace of this `DimensionUniverse`, combined with the version
89 to provide universe safety for registries that use different
90 dimension definitions.
91 builder : `DimensionConstructionBuilder`, optional
92 Builder object used to initialize a new instance. Ignored if
93 ``version`` is provided and an instance with that version already
94 exists. Should not have had `~DimensionConstructionBuilder.finish`
95 called; this will be called if needed by `DimensionUniverse`.
96 """
98 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {}
99 """Singleton dictionary of all instances, keyed by version.
101 For internal use only.
102 """
104 def __new__(
105 cls,
106 config: Optional[Config] = None,
107 *,
108 version: Optional[int] = None,
109 namespace: Optional[str] = None,
110 builder: Optional[DimensionConstructionBuilder] = None,
111 ) -> DimensionUniverse:
112 # Try to get a version first, to look for existing instances; try to
113 # do as little work as possible at this stage.
114 if version is None:
115 if builder is None:
116 config = DimensionConfig(config)
117 version = config["version"]
118 else:
119 version = builder.version
121 # Then a namespace.
122 if namespace is None:
123 if builder is None:
124 config = DimensionConfig(config)
125 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
126 else:
127 namespace = builder.namespace
128 # if still None use the default
129 if namespace is None:
130 namespace = _DEFAULT_NAMESPACE
132 # See if an equivalent instance already exists.
133 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace))
134 if self is not None:
135 return self
137 # Ensure we have a builder, building one from config if necessary.
138 if builder is None:
139 config = DimensionConfig(config)
140 builder = config.makeBuilder()
142 # Delegate to the builder for most of the construction work.
143 builder.finish()
145 # Create the universe instance and create core attributes, mostly
146 # copying from builder.
147 self = object.__new__(cls)
148 assert self is not None
149 self._cache = {}
150 self._dimensions = builder.dimensions
151 self._elements = builder.elements
152 self._topology = builder.topology
153 self._packers = builder.packers
154 self.dimensionConfig = builder.config
155 commonSkyPix = self._dimensions[builder.commonSkyPixName]
156 assert isinstance(commonSkyPix, SkyPixDimension)
157 self.commonSkyPix = commonSkyPix
159 # Attach self to all elements.
160 for element in self._elements:
161 element.universe = self
163 # Add attribute for special subsets of the graph.
164 self.empty = DimensionGraph(self, (), conform=False)
166 # Use the version number and namespace from the config as a key in
167 # the singleton dict containing all instances; that will let us
168 # transfer dimension objects between processes using pickle without
169 # actually going through real initialization, as long as a universe
170 # with the same version and namespace has already been constructed in
171 # the receiving process.
172 self._version = version
173 self._namespace = namespace
174 cls._instances[self._version, self._namespace] = self
176 # Build mappings from element to index. These are used for
177 # topological-sort comparison operators in DimensionElement itself.
178 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
179 # Same for dimension to index, sorted topologically across required
180 # and implied. This is used for encode/decode.
181 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
183 return self
185 @property
186 def version(self) -> int:
187 """The version number of this universe.
189 Returns
190 -------
191 version : `int`
192 An integer representing the version number of this universe.
193 Uniquely defined when combined with the `namespace`.
194 """
195 return self._version
197 @property
198 def namespace(self) -> str:
199 """The namespace associated with this universe.
201 Returns
202 -------
203 namespace : `str`
204 The namespace. When combined with the `version` can uniquely
205 define this universe.
206 """
207 return self._namespace
209 def __repr__(self) -> str:
210 return f"DimensionUniverse({self._version}, {self._namespace})"
212 def __getitem__(self, name: str) -> DimensionElement:
213 return self._elements[name]
215 def __contains__(self, name: Any) -> bool:
216 return name in self._elements
218 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
219 """Return the `DimensionElement` with the given name or a default.
221 Parameters
222 ----------
223 name : `str`
224 Name of the element.
225 default : `DimensionElement`, optional
226 Element to return if the named one does not exist. Defaults to
227 `None`.
229 Returns
230 -------
231 element : `DimensionElement`
232 The named element.
233 """
234 return self._elements.get(name, default)
236 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
237 """Return a set of all static elements in this universe.
239 Non-static elements that are created as needed may also exist, but
240 these are guaranteed to have no direct relationships to other elements
241 (though they may have spatial or temporal relationships).
243 Returns
244 -------
245 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
246 A frozen set of `DimensionElement` instances.
247 """
248 return self._elements
250 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
251 """Return a set of all static dimensions in this universe.
253 Non-static dimensions that are created as needed may also exist, but
254 these are guaranteed to have no direct relationships to other elements
255 (though they may have spatial or temporal relationships).
257 Returns
258 -------
259 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
260 A frozen set of `Dimension` instances.
261 """
262 return self._dimensions
264 @cached_getter
265 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
266 """Return a set of all `GovernorDimension` instances in this universe.
268 Returns
269 -------
270 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
271 A frozen set of `GovernorDimension` instances.
272 """
273 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
275 @cached_getter
276 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
277 """Return set of all `DatabaseDimensionElement` instances in universe.
279 This does not include `GovernorDimension` instances, which are backed
280 by the database but do not inherit from `DatabaseDimensionElement`.
282 Returns
283 -------
284 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
285 A frozen set of `DatabaseDimensionElement` instances.
286 """
287 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
289 @property # type: ignore
290 @cached_getter
291 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
292 """All skypix systems known to this universe.
294 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
295 """
296 return NamedValueSet(
297 [
298 family
299 for family in self._topology[TopologicalSpace.SPATIAL]
300 if isinstance(family, SkyPixSystem)
301 ]
302 ).freeze()
304 def getElementIndex(self, name: str) -> int:
305 """Return the position of the named dimension element.
307 The position is in this universe's sorting of all elements.
309 Parameters
310 ----------
311 name : `str`
312 Name of the element.
314 Returns
315 -------
316 index : `int`
317 Sorting index for this element.
318 """
319 return self._elementIndices[name]
321 def getDimensionIndex(self, name: str) -> int:
322 """Return the position of the named dimension.
324 This position is in this universe's sorting of all dimensions.
326 Parameters
327 ----------
328 name : `str`
329 Name of the dimension.
331 Returns
332 -------
333 index : `int`
334 Sorting index for this dimension.
336 Notes
337 -----
338 The dimension sort order for a universe is consistent with the element
339 order (all dimensions are elements), and either can be used to sort
340 dimensions if used consistently. But there are also some contexts in
341 which contiguous dimension-only indices are necessary or at least
342 desirable.
343 """
344 return self._dimensionIndices[name]
346 def expandDimensionNameSet(self, names: Set[str]) -> None:
347 """Expand a set of dimension names in-place.
349 Includes recursive dependencies.
351 This is an advanced interface for cases where constructing a
352 `DimensionGraph` (which also expands required dependencies) is
353 impossible or undesirable.
355 Parameters
356 ----------
357 names : `set` [ `str` ]
358 A true `set` of dimension names, to be expanded in-place.
359 """
360 # Keep iterating until the set of names stops growing. This is not as
361 # efficient as it could be, but we work pretty hard cache
362 # DimensionGraph instances to keep actual construction rare, so that
363 # shouldn't matter.
364 oldSize = len(names)
365 while True:
366 # iterate over a temporary copy so we can modify the original
367 for name in tuple(names):
368 names.update(self._dimensions[name].required.names)
369 names.update(self._dimensions[name].implied.names)
370 if oldSize == len(names):
371 break
372 else:
373 oldSize = len(names)
375 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
376 """Construct graph from iterable.
378 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
379 of `Dimension` instances and string names thereof.
381 Constructing `DimensionGraph` directly from names or dimension
382 instances is slightly more efficient when it is known in advance that
383 the iterable is not heterogenous.
385 Parameters
386 ----------
387 iterable: iterable of `Dimension` or `str`
388 Dimensions that must be included in the returned graph (their
389 dependencies will be as well).
391 Returns
392 -------
393 graph : `DimensionGraph`
394 A `DimensionGraph` instance containing all given dimensions.
395 """
396 names = set()
397 for item in iterable:
398 try:
399 names.add(item.name) # type: ignore
400 except AttributeError:
401 names.add(item)
402 return DimensionGraph(universe=self, names=names)
404 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
405 """Return a sorted version of the given iterable of dimension elements.
407 The universe's sort order is topological (an element's dependencies
408 precede it), with an unspecified (but deterministic) approach to
409 breaking ties.
411 Parameters
412 ----------
413 elements : iterable of `DimensionElement`.
414 Elements to be sorted.
415 reverse : `bool`, optional
416 If `True`, sort in the opposite order.
418 Returns
419 -------
420 sorted : `list` of `DimensionElement`
421 A sorted list containing the same elements that were given.
422 """
423 s = set(elements)
424 result = [element for element in self._elements if element in s or element.name in s]
425 if reverse:
426 result.reverse()
427 # mypy thinks this can return DimensionElements even if all the user
428 # passed it was Dimensions; we know better.
429 return result # type: ignore
431 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
432 """Make a dimension packer.
434 Constructs a `DimensionPacker` that can pack data ID dictionaries
435 into unique integers.
437 Parameters
438 ----------
439 name : `str`
440 Name of the packer, matching a key in the "packers" section of the
441 dimension configuration.
442 dataId : `DataCoordinate`
443 Fully-expanded data ID that identifies the at least the "fixed"
444 dimensions of the packer (i.e. those that are assumed/given,
445 setting the space over which packed integer IDs are unique).
446 ``dataId.hasRecords()`` must return `True`.
447 """
448 return self._packers[name](self, dataId)
450 def getEncodeLength(self) -> int:
451 """Return encoded size of graph.
453 Returns the size (in bytes) of the encoded size of `DimensionGraph`
454 instances in this universe.
456 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
457 information.
458 """
459 return math.ceil(len(self._dimensions) / 8)
461 @classmethod
462 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse:
463 """Return an unpickled dimension universe.
465 Callable used for unpickling.
467 For internal use only.
468 """
469 if namespace is None:
470 # Old pickled universe.
471 namespace = _DEFAULT_NAMESPACE
472 try:
473 return cls._instances[version, namespace]
474 except KeyError as err:
475 raise pickle.UnpicklingError(
476 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
477 f"not found. Note that DimensionUniverse objects are not "
478 f"truly serialized; when using pickle to transfer them "
479 f"between processes, an equivalent instance with the same "
480 f"version must already exist in the receiving process."
481 ) from err
483 def __reduce__(self) -> tuple:
484 return (self._unpickle, (self._version, self._namespace))
486 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
487 # DimensionUniverse is recursively immutable; see note in @immutable
488 # decorator.
489 return self
491 # Class attributes below are shadowed by instance attributes, and are
492 # present just to hold the docstrings for those instance attributes.
494 empty: DimensionGraph
495 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
496 """
498 commonSkyPix: SkyPixDimension
499 """The special skypix dimension that is used to relate all other spatial
500 dimensions in the `Registry` database (`SkyPixDimension`).
501 """
503 dimensionConfig: DimensionConfig
504 """The configuration used to create this Universe (`DimensionConfig`)."""
506 _cache: Dict[FrozenSet[str], DimensionGraph]
508 _dimensions: NamedValueAbstractSet[Dimension]
510 _elements: NamedValueAbstractSet[DimensionElement]
512 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
514 _dimensionIndices: Dict[str, int]
516 _elementIndices: Dict[str, int]
518 _packers: Dict[str, DimensionPackerFactory]
520 _version: int
522 _namespace: str