Coverage for python/lsst/daf/butler/core/dimensions/_universe.py : 36%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 ClassVar,
30 Dict,
31 FrozenSet,
32 Iterable,
33 List,
34 Mapping,
35 Optional,
36 Set,
37 TYPE_CHECKING,
38 TypeVar,
39 Union,
40)
42from ..config import Config
43from ..named import NamedValueAbstractSet, NamedValueSet
44from .._topology import TopologicalSpace, TopologicalFamily
45from ..utils import cached_getter, immutable
46from ._config import DimensionConfig
47from ._elements import Dimension, DimensionElement
48from ._graph import DimensionGraph
49from ._governor import GovernorDimension
50from ._database import DatabaseDimensionElement
51from ._skypix import SkyPixDimension, SkyPixSystem
53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true
54 from ._coordinate import DataCoordinate
55 from ._packer import DimensionPacker, DimensionPackerFactory
56 from .construction import DimensionConstructionBuilder
59E = TypeVar("E", bound=DimensionElement)
62@immutable
63class DimensionUniverse:
64 """Self-consistent set of dimensions.
66 A parent class that represents a complete, self-consistent set of
67 dimensions and their relationships.
69 `DimensionUniverse` is not a class-level singleton, but all instances are
70 tracked in a singleton map keyed by the version number in the configuration
71 they were loaded from. Because these universes are solely responsible for
72 constructing `DimensionElement` instances, these are also indirectly
73 tracked by that singleton as well.
75 Parameters
76 ----------
77 config : `Config`, optional
78 Configuration object from which dimension definitions can be extracted.
79 Ignored if ``builder`` is provided, or if ``version`` is provided and
80 an instance with that version already exists.
81 version : `int`, optional
82 Integer version for this `DimensionUniverse`. If not provided, a
83 version will be obtained from ``builder`` or ``config``.
84 builder : `DimensionConstructionBuilder`, optional
85 Builder object used to initialize a new instance. Ignored if
86 ``version`` is provided and an instance with that version already
87 exists. Should not have had `~DimensionConstructionBuilder.finish`
88 called; this will be called if needed by `DimensionUniverse`.
89 """
91 _instances: ClassVar[Dict[int, DimensionUniverse]] = {}
92 """Singleton dictionary of all instances, keyed by version.
94 For internal use only.
95 """
97 def __new__(
98 cls,
99 config: Optional[Config] = None, *,
100 version: Optional[int] = None,
101 builder: Optional[DimensionConstructionBuilder] = None,
102 ) -> DimensionUniverse:
103 # Try to get a version first, to look for existing instances; try to
104 # do as little work as possible at this stage.
105 if version is None:
106 if builder is None:
107 config = DimensionConfig(config)
108 version = config["version"]
109 else:
110 version = builder.version
112 # See if an equivalent instance already exists.
113 self: Optional[DimensionUniverse] = cls._instances.get(version)
114 if self is not None:
115 return self
117 # Ensure we have a builder, building one from config if necessary.
118 if builder is None:
119 config = DimensionConfig(config)
120 builder = config.makeBuilder()
122 # Delegate to the builder for most of the construction work.
123 builder.finish()
125 # Create the universe instance and create core attributes, mostly
126 # copying from builder.
127 self = object.__new__(cls)
128 assert self is not None
129 self._cache = {}
130 self._dimensions = builder.dimensions
131 self._elements = builder.elements
132 self._topology = builder.topology
133 self._packers = builder.packers
134 self.dimensionConfig = builder.config
135 commonSkyPix = self._dimensions[builder.commonSkyPixName]
136 assert isinstance(commonSkyPix, SkyPixDimension)
137 self.commonSkyPix = commonSkyPix
139 # Attach self to all elements.
140 for element in self._elements:
141 element.universe = self
143 # Add attribute for special subsets of the graph.
144 self.empty = DimensionGraph(self, (), conform=False)
146 # Use the version number from the config as a key in the singleton
147 # dict containing all instances; that will let us transfer dimension
148 # objects between processes using pickle without actually going
149 # through real initialization, as long as a universe with the same
150 # version has already been constructed in the receiving process.
151 self._version = version
152 cls._instances[self._version] = self
154 # Build mappings from element to index. These are used for
155 # topological-sort comparison operators in DimensionElement itself.
156 self._elementIndices = {
157 name: i for i, name in enumerate(self._elements.names)
158 }
159 # Same for dimension to index, sorted topologically across required
160 # and implied. This is used for encode/decode.
161 self._dimensionIndices = {
162 name: i for i, name in enumerate(self._dimensions.names)
163 }
165 return self
167 def __repr__(self) -> str:
168 return f"DimensionUniverse({self._version})"
170 def __getitem__(self, name: str) -> DimensionElement:
171 return self._elements[name]
173 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
174 """Return the `DimensionElement` with the given name or a default.
176 Parameters
177 ----------
178 name : `str`
179 Name of the element.
180 default : `DimensionElement`, optional
181 Element to return if the named one does not exist. Defaults to
182 `None`.
184 Returns
185 -------
186 element : `DimensionElement`
187 The named element.
188 """
189 return self._elements.get(name, default)
191 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
192 """Return a set of all static elements in this universe.
194 Non-static elements that are created as needed may also exist, but
195 these are guaranteed to have no direct relationships to other elements
196 (though they may have spatial or temporal relationships).
198 Returns
199 -------
200 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
201 A frozen set of `DimensionElement` instances.
202 """
203 return self._elements
205 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
206 """Return a set of all static dimensions in this universe.
208 Non-static dimensions that are created as needed may also exist, but
209 these are guaranteed to have no direct relationships to other elements
210 (though they may have spatial or temporal relationships).
212 Returns
213 -------
214 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
215 A frozen set of `Dimension` instances.
216 """
217 return self._dimensions
219 @cached_getter
220 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
221 """Return a set of all `GovernorDimension` instances in this universe.
223 Returns
224 -------
225 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
226 A frozen set of `GovernorDimension` instances.
227 """
228 return NamedValueSet(
229 d for d in self._dimensions if isinstance(d, GovernorDimension)
230 ).freeze()
232 @cached_getter
233 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
234 """Return set of all `DatabaseDimensionElement` instances in universe.
236 This does not include `GovernorDimension` instances, which are backed
237 by the database but do not inherit from `DatabaseDimensionElement`.
239 Returns
240 -------
241 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
242 A frozen set of `DatabaseDimensionElement` instances.
243 """
244 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
246 @property # type: ignore
247 @cached_getter
248 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
249 """All skypix systems known to this universe.
251 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
252 """
253 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL]
254 if isinstance(family, SkyPixSystem)]).freeze()
256 def getElementIndex(self, name: str) -> int:
257 """Return the position of the named dimension element.
259 The position is in this universe's sorting of all elements.
261 Parameters
262 ----------
263 name : `str`
264 Name of the element.
266 Returns
267 -------
268 index : `int`
269 Sorting index for this element.
270 """
271 return self._elementIndices[name]
273 def getDimensionIndex(self, name: str) -> int:
274 """Return the position of the named dimension.
276 This position is in this universe's sorting of all dimensions.
278 Parameters
279 ----------
280 name : `str`
281 Name of the dimension.
283 Returns
284 -------
285 index : `int`
286 Sorting index for this dimension.
288 Notes
289 -----
290 The dimension sort order for a universe is consistent with the element
291 order (all dimensions are elements), and either can be used to sort
292 dimensions if used consistently. But there are also some contexts in
293 which contiguous dimension-only indices are necessary or at least
294 desirable.
295 """
296 return self._dimensionIndices[name]
298 def expandDimensionNameSet(self, names: Set[str]) -> None:
299 """Expand a set of dimension names in-place.
301 Includes recursive dependencies.
303 This is an advanced interface for cases where constructing a
304 `DimensionGraph` (which also expands required dependencies) is
305 impossible or undesirable.
307 Parameters
308 ----------
309 names : `set` [ `str` ]
310 A true `set` of dimension names, to be expanded in-place.
311 """
312 # Keep iterating until the set of names stops growing. This is not as
313 # efficient as it could be, but we work pretty hard cache
314 # DimensionGraph instances to keep actual construction rare, so that
315 # shouldn't matter.
316 oldSize = len(names)
317 while True:
318 # iterate over a temporary copy so we can modify the original
319 for name in tuple(names):
320 names.update(self._dimensions[name].required.names)
321 names.update(self._dimensions[name].implied.names)
322 if oldSize == len(names):
323 break
324 else:
325 oldSize = len(names)
327 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
328 """Construct graph from iterable.
330 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
331 of `Dimension` instances and string names thereof.
333 Constructing `DimensionGraph` directly from names or dimension
334 instances is slightly more efficient when it is known in advance that
335 the iterable is not heterogenous.
337 Parameters
338 ----------
339 iterable: iterable of `Dimension` or `str`
340 Dimensions that must be included in the returned graph (their
341 dependencies will be as well).
343 Returns
344 -------
345 graph : `DimensionGraph`
346 A `DimensionGraph` instance containing all given dimensions.
347 """
348 names = set()
349 for item in iterable:
350 try:
351 names.add(item.name) # type: ignore
352 except AttributeError:
353 names.add(item)
354 return DimensionGraph(universe=self, names=names)
356 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
357 """Return a sorted version of the given iterable of dimension elements.
359 The universe's sort order is topological (an element's dependencies
360 precede it), with an unspecified (but deterministic) approach to
361 breaking ties.
363 Parameters
364 ----------
365 elements : iterable of `DimensionElement`.
366 Elements to be sorted.
367 reverse : `bool`, optional
368 If `True`, sort in the opposite order.
370 Returns
371 -------
372 sorted : `list` of `DimensionElement`
373 A sorted list containing the same elements that were given.
374 """
375 s = set(elements)
376 result = [element for element in self._elements if element in s or element.name in s]
377 if reverse:
378 result.reverse()
379 # mypy thinks this can return DimensionElements even if all the user
380 # passed it was Dimensions; we know better.
381 return result # type: ignore
383 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
384 """Make a dimension packer.
386 Constructs a `DimensionPacker` that can pack data ID dictionaries
387 into unique integers.
389 Parameters
390 ----------
391 name : `str`
392 Name of the packer, matching a key in the "packers" section of the
393 dimension configuration.
394 dataId : `DataCoordinate`
395 Fully-expanded data ID that identfies the at least the "fixed"
396 dimensions of the packer (i.e. those that are assumed/given,
397 setting the space over which packed integer IDs are unique).
398 ``dataId.hasRecords()`` must return `True`.
399 """
400 return self._packers[name](self, dataId)
402 def getEncodeLength(self) -> int:
403 """Return encoded size of graph.
405 Returns the size (in bytes) of the encoded size of `DimensionGraph`
406 instances in this universe.
408 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
409 information.
410 """
411 return math.ceil(len(self._dimensions)/8)
413 @classmethod
414 def _unpickle(cls, version: int) -> DimensionUniverse:
415 """Return an unpickled dimension universe.
417 Callable used for unpickling.
419 For internal use only.
420 """
421 try:
422 return cls._instances[version]
423 except KeyError as err:
424 raise pickle.UnpicklingError(
425 f"DimensionUniverse with version '{version}' "
426 f"not found. Note that DimensionUniverse objects are not "
427 f"truly serialized; when using pickle to transfer them "
428 f"between processes, an equivalent instance with the same "
429 f"version must already exist in the receiving process."
430 ) from err
432 def __reduce__(self) -> tuple:
433 return (self._unpickle, (self._version,))
435 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
436 # DimensionUniverse is recursively immutable; see note in @immutable
437 # decorator.
438 return self
440 # Class attributes below are shadowed by instance attributes, and are
441 # present just to hold the docstrings for those instance attributes.
443 empty: DimensionGraph
444 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
445 """
447 commonSkyPix: SkyPixDimension
448 """The special skypix dimension that is used to relate all other spatial
449 dimensions in the `Registry` database (`SkyPixDimension`).
450 """
452 dimensionConfig: DimensionConfig
453 """The configuration used to create this Universe (`DimensionConfig`)."""
455 _cache: Dict[FrozenSet[str], DimensionGraph]
457 _dimensions: NamedValueAbstractSet[Dimension]
459 _elements: NamedValueAbstractSet[DimensionElement]
461 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
463 _dimensionIndices: Dict[str, int]
465 _elementIndices: Dict[str, int]
467 _packers: Dict[str, DimensionPackerFactory]
469 _version: int