Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 35%
158 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:01 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:01 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import logging
27import math
28import pickle
29from typing import (
30 TYPE_CHECKING,
31 Any,
32 ClassVar,
33 Dict,
34 FrozenSet,
35 Iterable,
36 List,
37 Mapping,
38 Optional,
39 Set,
40 Tuple,
41 TypeVar,
42 Union,
43)
45from lsst.utils.classes import cached_getter, immutable
47from .._topology import TopologicalFamily, TopologicalSpace
48from ..config import Config
49from ..named import NamedValueAbstractSet, NamedValueSet
50from ._config import _DEFAULT_NAMESPACE, DimensionConfig
51from ._database import DatabaseDimensionElement
52from ._elements import Dimension, DimensionElement
53from ._governor import GovernorDimension
54from ._graph import DimensionGraph
55from ._skypix import SkyPixDimension, SkyPixSystem
57if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 57 ↛ 58line 57 didn't jump to line 58, because the condition on line 57 was never true
58 from ._coordinate import DataCoordinate
59 from ._packer import DimensionPacker, DimensionPackerFactory
60 from .construction import DimensionConstructionBuilder
63E = TypeVar("E", bound=DimensionElement)
64_LOG = logging.getLogger(__name__)
67@immutable
68class DimensionUniverse:
69 """Self-consistent set of dimensions.
71 A parent class that represents a complete, self-consistent set of
72 dimensions and their relationships.
74 `DimensionUniverse` is not a class-level singleton, but all instances are
75 tracked in a singleton map keyed by the version number and namespace
76 in the configuration they were loaded from. Because these universes
77 are solely responsible for constructing `DimensionElement` instances,
78 these are also indirectly tracked by that singleton as well.
80 Parameters
81 ----------
82 config : `Config`, optional
83 Configuration object from which dimension definitions can be extracted.
84 Ignored if ``builder`` is provided, or if ``version`` is provided and
85 an instance with that version already exists.
86 version : `int`, optional
87 Integer version for this `DimensionUniverse`. If not provided, a
88 version will be obtained from ``builder`` or ``config``.
89 namespace : `str`, optional
90 Namespace of this `DimensionUniverse`, combined with the version
91 to provide universe safety for registries that use different
92 dimension definitions.
93 builder : `DimensionConstructionBuilder`, optional
94 Builder object used to initialize a new instance. Ignored if
95 ``version`` is provided and an instance with that version already
96 exists. Should not have had `~DimensionConstructionBuilder.finish`
97 called; this will be called if needed by `DimensionUniverse`.
98 """
100 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {}
101 """Singleton dictionary of all instances, keyed by version.
103 For internal use only.
104 """
106 def __new__(
107 cls,
108 config: Optional[Config] = None,
109 *,
110 version: Optional[int] = None,
111 namespace: Optional[str] = None,
112 builder: Optional[DimensionConstructionBuilder] = None,
113 ) -> DimensionUniverse:
114 # Try to get a version first, to look for existing instances; try to
115 # do as little work as possible at this stage.
116 if version is None:
117 if builder is None:
118 config = DimensionConfig(config)
119 version = config["version"]
120 else:
121 version = builder.version
123 # Then a namespace.
124 if namespace is None:
125 if builder is None:
126 config = DimensionConfig(config)
127 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
128 else:
129 namespace = builder.namespace
130 # if still None use the default
131 if namespace is None:
132 namespace = _DEFAULT_NAMESPACE
134 # See if an equivalent instance already exists.
135 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace))
136 if self is not None:
137 return self
139 # Ensure we have a builder, building one from config if necessary.
140 if builder is None:
141 config = DimensionConfig(config)
142 builder = config.makeBuilder()
144 # Delegate to the builder for most of the construction work.
145 builder.finish()
147 # Create the universe instance and create core attributes, mostly
148 # copying from builder.
149 self = object.__new__(cls)
150 assert self is not None
151 self._cache = {}
152 self._dimensions = builder.dimensions
153 self._elements = builder.elements
154 self._topology = builder.topology
155 self._packers = builder.packers
156 self.dimensionConfig = builder.config
157 commonSkyPix = self._dimensions[builder.commonSkyPixName]
158 assert isinstance(commonSkyPix, SkyPixDimension)
159 self.commonSkyPix = commonSkyPix
161 # Attach self to all elements.
162 for element in self._elements:
163 element.universe = self
165 # Add attribute for special subsets of the graph.
166 self.empty = DimensionGraph(self, (), conform=False)
168 # Use the version number and namespace from the config as a key in
169 # the singleton dict containing all instances; that will let us
170 # transfer dimension objects between processes using pickle without
171 # actually going through real initialization, as long as a universe
172 # with the same version and namespace has already been constructed in
173 # the receiving process.
174 self._version = version
175 self._namespace = namespace
176 cls._instances[self._version, self._namespace] = self
178 # Build mappings from element to index. These are used for
179 # topological-sort comparison operators in DimensionElement itself.
180 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
181 # Same for dimension to index, sorted topologically across required
182 # and implied. This is used for encode/decode.
183 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
185 return self
187 @property
188 def version(self) -> int:
189 """The version number of this universe.
191 Returns
192 -------
193 version : `int`
194 An integer representing the version number of this universe.
195 Uniquely defined when combined with the `namespace`.
196 """
197 return self._version
199 @property
200 def namespace(self) -> str:
201 """The namespace associated with this universe.
203 Returns
204 -------
205 namespace : `str`
206 The namespace. When combined with the `version` can uniquely
207 define this universe.
208 """
209 return self._namespace
211 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
212 """Check compatibility between this `DimensionUniverse` and another
214 Parameters
215 ----------
216 other : `DimensionUniverse`
217 The other `DimensionUniverse` to check for compatibility
219 Returns
220 -------
221 results : `bool`
222 If the other `DimensionUniverse` is compatible with this one return
223 `True`, else `False`
224 """
225 # Different namespaces mean that these universes cannot be compatible.
226 if self.namespace != other.namespace:
227 return False
228 if self.version != other.version:
229 _LOG.info(
230 "Universes share a namespace %r but have differing versions (%d != %d). "
231 " This could be okay but may be responsible for dimension errors later.",
232 self.namespace,
233 self.version,
234 other.version,
235 )
237 # For now assume compatibility if versions differ.
238 return True
240 def __repr__(self) -> str:
241 return f"DimensionUniverse({self._version}, {self._namespace})"
243 def __getitem__(self, name: str) -> DimensionElement:
244 return self._elements[name]
246 def __contains__(self, name: Any) -> bool:
247 return name in self._elements
249 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
250 """Return the `DimensionElement` with the given name or a default.
252 Parameters
253 ----------
254 name : `str`
255 Name of the element.
256 default : `DimensionElement`, optional
257 Element to return if the named one does not exist. Defaults to
258 `None`.
260 Returns
261 -------
262 element : `DimensionElement`
263 The named element.
264 """
265 return self._elements.get(name, default)
267 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
268 """Return a set of all static elements in this universe.
270 Non-static elements that are created as needed may also exist, but
271 these are guaranteed to have no direct relationships to other elements
272 (though they may have spatial or temporal relationships).
274 Returns
275 -------
276 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
277 A frozen set of `DimensionElement` instances.
278 """
279 return self._elements
281 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
282 """Return a set of all static dimensions in this universe.
284 Non-static dimensions that are created as needed may also exist, but
285 these are guaranteed to have no direct relationships to other elements
286 (though they may have spatial or temporal relationships).
288 Returns
289 -------
290 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
291 A frozen set of `Dimension` instances.
292 """
293 return self._dimensions
295 @cached_getter
296 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
297 """Return a set of all `GovernorDimension` instances in this universe.
299 Returns
300 -------
301 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
302 A frozen set of `GovernorDimension` instances.
303 """
304 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
306 @cached_getter
307 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
308 """Return set of all `DatabaseDimensionElement` instances in universe.
310 This does not include `GovernorDimension` instances, which are backed
311 by the database but do not inherit from `DatabaseDimensionElement`.
313 Returns
314 -------
315 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
316 A frozen set of `DatabaseDimensionElement` instances.
317 """
318 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
320 @property
321 @cached_getter
322 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
323 """All skypix systems known to this universe.
325 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
326 """
327 return NamedValueSet(
328 [
329 family
330 for family in self._topology[TopologicalSpace.SPATIAL]
331 if isinstance(family, SkyPixSystem)
332 ]
333 ).freeze()
335 def getElementIndex(self, name: str) -> int:
336 """Return the position of the named dimension element.
338 The position is in this universe's sorting of all elements.
340 Parameters
341 ----------
342 name : `str`
343 Name of the element.
345 Returns
346 -------
347 index : `int`
348 Sorting index for this element.
349 """
350 return self._elementIndices[name]
352 def getDimensionIndex(self, name: str) -> int:
353 """Return the position of the named dimension.
355 This position is in this universe's sorting of all dimensions.
357 Parameters
358 ----------
359 name : `str`
360 Name of the dimension.
362 Returns
363 -------
364 index : `int`
365 Sorting index for this dimension.
367 Notes
368 -----
369 The dimension sort order for a universe is consistent with the element
370 order (all dimensions are elements), and either can be used to sort
371 dimensions if used consistently. But there are also some contexts in
372 which contiguous dimension-only indices are necessary or at least
373 desirable.
374 """
375 return self._dimensionIndices[name]
377 def expandDimensionNameSet(self, names: Set[str]) -> None:
378 """Expand a set of dimension names in-place.
380 Includes recursive dependencies.
382 This is an advanced interface for cases where constructing a
383 `DimensionGraph` (which also expands required dependencies) is
384 impossible or undesirable.
386 Parameters
387 ----------
388 names : `set` [ `str` ]
389 A true `set` of dimension names, to be expanded in-place.
390 """
391 # Keep iterating until the set of names stops growing. This is not as
392 # efficient as it could be, but we work pretty hard cache
393 # DimensionGraph instances to keep actual construction rare, so that
394 # shouldn't matter.
395 oldSize = len(names)
396 while True:
397 # iterate over a temporary copy so we can modify the original
398 for name in tuple(names):
399 names.update(self._dimensions[name].required.names)
400 names.update(self._dimensions[name].implied.names)
401 if oldSize == len(names):
402 break
403 else:
404 oldSize = len(names)
406 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
407 """Construct graph from iterable.
409 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
410 of `Dimension` instances and string names thereof.
412 Constructing `DimensionGraph` directly from names or dimension
413 instances is slightly more efficient when it is known in advance that
414 the iterable is not heterogenous.
416 Parameters
417 ----------
418 iterable: iterable of `Dimension` or `str`
419 Dimensions that must be included in the returned graph (their
420 dependencies will be as well).
422 Returns
423 -------
424 graph : `DimensionGraph`
425 A `DimensionGraph` instance containing all given dimensions.
426 """
427 names = set()
428 for item in iterable:
429 try:
430 names.add(item.name) # type: ignore
431 except AttributeError:
432 names.add(item)
433 return DimensionGraph(universe=self, names=names)
435 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
436 """Return a sorted version of the given iterable of dimension elements.
438 The universe's sort order is topological (an element's dependencies
439 precede it), with an unspecified (but deterministic) approach to
440 breaking ties.
442 Parameters
443 ----------
444 elements : iterable of `DimensionElement`.
445 Elements to be sorted.
446 reverse : `bool`, optional
447 If `True`, sort in the opposite order.
449 Returns
450 -------
451 sorted : `list` of `DimensionElement`
452 A sorted list containing the same elements that were given.
453 """
454 s = set(elements)
455 result = [element for element in self._elements if element in s or element.name in s]
456 if reverse:
457 result.reverse()
458 # mypy thinks this can return DimensionElements even if all the user
459 # passed it was Dimensions; we know better.
460 return result # type: ignore
462 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
463 """Make a dimension packer.
465 Constructs a `DimensionPacker` that can pack data ID dictionaries
466 into unique integers.
468 Parameters
469 ----------
470 name : `str`
471 Name of the packer, matching a key in the "packers" section of the
472 dimension configuration.
473 dataId : `DataCoordinate`
474 Fully-expanded data ID that identifies the at least the "fixed"
475 dimensions of the packer (i.e. those that are assumed/given,
476 setting the space over which packed integer IDs are unique).
477 ``dataId.hasRecords()`` must return `True`.
478 """
479 return self._packers[name](self, dataId)
481 def getEncodeLength(self) -> int:
482 """Return encoded size of graph.
484 Returns the size (in bytes) of the encoded size of `DimensionGraph`
485 instances in this universe.
487 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
488 information.
489 """
490 return math.ceil(len(self._dimensions) / 8)
492 @classmethod
493 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse:
494 """Return an unpickled dimension universe.
496 Callable used for unpickling.
498 For internal use only.
499 """
500 if namespace is None:
501 # Old pickled universe.
502 namespace = _DEFAULT_NAMESPACE
503 try:
504 return cls._instances[version, namespace]
505 except KeyError as err:
506 raise pickle.UnpicklingError(
507 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
508 f"not found. Note that DimensionUniverse objects are not "
509 f"truly serialized; when using pickle to transfer them "
510 f"between processes, an equivalent instance with the same "
511 f"version must already exist in the receiving process."
512 ) from err
514 def __reduce__(self) -> tuple:
515 return (self._unpickle, (self._version, self._namespace))
517 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
518 # DimensionUniverse is recursively immutable; see note in @immutable
519 # decorator.
520 return self
522 # Class attributes below are shadowed by instance attributes, and are
523 # present just to hold the docstrings for those instance attributes.
525 empty: DimensionGraph
526 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
527 """
529 commonSkyPix: SkyPixDimension
530 """The special skypix dimension that is used to relate all other spatial
531 dimensions in the `Registry` database (`SkyPixDimension`).
532 """
534 dimensionConfig: DimensionConfig
535 """The configuration used to create this Universe (`DimensionConfig`)."""
537 _cache: Dict[FrozenSet[str], DimensionGraph]
539 _dimensions: NamedValueAbstractSet[Dimension]
541 _elements: NamedValueAbstractSet[DimensionElement]
543 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
545 _dimensionIndices: Dict[str, int]
547 _elementIndices: Dict[str, int]
549 _packers: Dict[str, DimensionPackerFactory]
551 _version: int
553 _namespace: str