Coverage for python/lsst/daf/butler/core/dimensions/_universe.py : 36%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 ClassVar,
30 Dict,
31 FrozenSet,
32 Iterable,
33 List,
34 Mapping,
35 Optional,
36 Set,
37 TYPE_CHECKING,
38 TypeVar,
39 Union,
40)
42from ..config import Config
43from ..named import NamedValueAbstractSet, NamedValueSet
44from .._topology import TopologicalSpace, TopologicalFamily
45from ..utils import cached_getter, immutable
46from ._config import DimensionConfig
47from ._elements import Dimension, DimensionElement
48from ._graph import DimensionGraph
49from ._governor import GovernorDimension
50from ._database import DatabaseDimensionElement
51from ._skypix import SkyPixDimension, SkyPixSystem
53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true
54 from ._coordinate import DataCoordinate
55 from ._packer import DimensionPacker, DimensionPackerFactory
56 from .construction import DimensionConstructionBuilder
59E = TypeVar("E", bound=DimensionElement)
62@immutable
63class DimensionUniverse:
64 """Self-consistent set of dimensions.
66 A parent class that represents a complete, self-consistent set of
67 dimensions and their relationships.
69 `DimensionUniverse` is not a class-level singleton, but all instances are
70 tracked in a singleton map keyed by the version number in the configuration
71 they were loaded from. Because these universes are solely responsible for
72 constructing `DimensionElement` instances, these are also indirectly
73 tracked by that singleton as well.
75 Parameters
76 ----------
77 config : `Config`, optional
78 Configuration object from which dimension definitions can be extracted.
79 Ignored if ``builder`` is provided, or if ``version`` is provided and
80 an instance with that version already exists.
81 version : `int`, optional
82 Integer version for this `DimensionUniverse`. If not provided, a
83 version will be obtained from ``builder`` or ``config``.
84 builder : `DimensionConstructionBuilder`, optional
85 Builder object used to initialize a new instance. Ignored if
86 ``version`` is provided and an instance with that version already
87 exists. Should not have had `~DimensionConstructionBuilder.finish`
88 called; this will be called if needed by `DimensionUniverse`.
89 """
91 _instances: ClassVar[Dict[int, DimensionUniverse]] = {}
92 """Singleton dictionary of all instances, keyed by version.
94 For internal use only.
95 """
97 def __new__(
98 cls,
99 config: Optional[Config] = None, *,
100 version: Optional[int] = None,
101 builder: Optional[DimensionConstructionBuilder] = None,
102 ) -> DimensionUniverse:
103 # Try to get a version first, to look for existing instances; try to
104 # do as little work as possible at this stage.
105 if version is None:
106 if builder is None:
107 config = DimensionConfig(config)
108 version = config["version"]
109 else:
110 version = builder.version
112 # See if an equivalent instance already exists.
113 self: Optional[DimensionUniverse] = cls._instances.get(version)
114 if self is not None:
115 return self
117 # Ensure we have a builder, building one from config if necessary.
118 if builder is None:
119 config = DimensionConfig(config)
120 builder = config.makeBuilder()
122 # Delegate to the builder for most of the construction work.
123 builder.finish()
125 # Create the universe instance and create core attributes, mostly
126 # copying from builder.
127 self = object.__new__(cls)
128 assert self is not None
129 self._cache = {}
130 self._dimensions = builder.dimensions
131 self._elements = builder.elements
132 self._topology = builder.topology
133 self._packers = builder.packers
134 commonSkyPix = self._dimensions[builder.commonSkyPixName]
135 assert isinstance(commonSkyPix, SkyPixDimension)
136 self.commonSkyPix = commonSkyPix
138 # Attach self to all elements.
139 for element in self._elements:
140 element.universe = self
142 # Add attribute for special subsets of the graph.
143 self.empty = DimensionGraph(self, (), conform=False)
145 # Use the version number from the config as a key in the singleton
146 # dict containing all instances; that will let us transfer dimension
147 # objects between processes using pickle without actually going
148 # through real initialization, as long as a universe with the same
149 # version has already been constructed in the receiving process.
150 self._version = version
151 cls._instances[self._version] = self
153 # Build mappings from element to index. These are used for
154 # topological-sort comparison operators in DimensionElement itself.
155 self._elementIndices = {
156 name: i for i, name in enumerate(self._elements.names)
157 }
158 # Same for dimension to index, sorted topologically across required
159 # and implied. This is used for encode/decode.
160 self._dimensionIndices = {
161 name: i for i, name in enumerate(self._dimensions.names)
162 }
164 return self
166 def __repr__(self) -> str:
167 return f"DimensionUniverse({self._version})"
169 def __getitem__(self, name: str) -> DimensionElement:
170 return self._elements[name]
172 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
173 """Return the `DimensionElement` with the given name or a default.
175 Parameters
176 ----------
177 name : `str`
178 Name of the element.
179 default : `DimensionElement`, optional
180 Element to return if the named one does not exist. Defaults to
181 `None`.
183 Returns
184 -------
185 element : `DimensionElement`
186 The named element.
187 """
188 return self._elements.get(name, default)
190 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
191 """Return a set of all static elements in this universe.
193 Non-static elements that are created as needed may also exist, but
194 these are guaranteed to have no direct relationships to other elements
195 (though they may have spatial or temporal relationships).
197 Returns
198 -------
199 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
200 A frozen set of `DimensionElement` instances.
201 """
202 return self._elements
204 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
205 """Return a set of all static dimensions in this universe.
207 Non-static dimensions that are created as needed may also exist, but
208 these are guaranteed to have no direct relationships to other elements
209 (though they may have spatial or temporal relationships).
211 Returns
212 -------
213 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
214 A frozen set of `Dimension` instances.
215 """
216 return self._dimensions
218 @cached_getter
219 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
220 """Return a set of all `GovernorDimension` instances in this universe.
222 Returns
223 -------
224 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
225 A frozen set of `GovernorDimension` instances.
226 """
227 return NamedValueSet(
228 d for d in self._dimensions if isinstance(d, GovernorDimension)
229 ).freeze()
231 @cached_getter
232 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
233 """Return set of all `DatabaseDimensionElement` instances in universe.
235 This does not include `GovernorDimension` instances, which are backed
236 by the database but do not inherit from `DatabaseDimensionElement`.
238 Returns
239 -------
240 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
241 A frozen set of `DatabaseDimensionElement` instances.
242 """
243 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
245 @property # type: ignore
246 @cached_getter
247 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
248 """All skypix systems known to this universe.
250 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
251 """
252 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL]
253 if isinstance(family, SkyPixSystem)]).freeze()
255 def getElementIndex(self, name: str) -> int:
256 """Return the position of the named dimension element.
258 The position is in this universe's sorting of all elements.
260 Parameters
261 ----------
262 name : `str`
263 Name of the element.
265 Returns
266 -------
267 index : `int`
268 Sorting index for this element.
269 """
270 return self._elementIndices[name]
272 def getDimensionIndex(self, name: str) -> int:
273 """Return the position of the named dimension.
275 This position is in this universe's sorting of all dimensions.
277 Parameters
278 ----------
279 name : `str`
280 Name of the dimension.
282 Returns
283 -------
284 index : `int`
285 Sorting index for this dimension.
287 Notes
288 -----
289 The dimension sort order for a universe is consistent with the element
290 order (all dimensions are elements), and either can be used to sort
291 dimensions if used consistently. But there are also some contexts in
292 which contiguous dimension-only indices are necessary or at least
293 desirable.
294 """
295 return self._dimensionIndices[name]
297 def expandDimensionNameSet(self, names: Set[str]) -> None:
298 """Expand a set of dimension names in-place.
300 Includes recursive dependencies.
302 This is an advanced interface for cases where constructing a
303 `DimensionGraph` (which also expands required dependencies) is
304 impossible or undesirable.
306 Parameters
307 ----------
308 names : `set` [ `str` ]
309 A true `set` of dimension names, to be expanded in-place.
310 """
311 # Keep iterating until the set of names stops growing. This is not as
312 # efficient as it could be, but we work pretty hard cache
313 # DimensionGraph instances to keep actual construction rare, so that
314 # shouldn't matter.
315 oldSize = len(names)
316 while True:
317 # iterate over a temporary copy so we can modify the original
318 for name in tuple(names):
319 names.update(self._dimensions[name].required.names)
320 names.update(self._dimensions[name].implied.names)
321 if oldSize == len(names):
322 break
323 else:
324 oldSize = len(names)
326 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
327 """Construct graph from iterable.
329 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
330 of `Dimension` instances and string names thereof.
332 Constructing `DimensionGraph` directly from names or dimension
333 instances is slightly more efficient when it is known in advance that
334 the iterable is not heterogenous.
336 Parameters
337 ----------
338 iterable: iterable of `Dimension` or `str`
339 Dimensions that must be included in the returned graph (their
340 dependencies will be as well).
342 Returns
343 -------
344 graph : `DimensionGraph`
345 A `DimensionGraph` instance containing all given dimensions.
346 """
347 names = set()
348 for item in iterable:
349 try:
350 names.add(item.name) # type: ignore
351 except AttributeError:
352 names.add(item)
353 return DimensionGraph(universe=self, names=names)
355 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
356 """Return a sorted version of the given iterable of dimension elements.
358 The universe's sort order is topological (an element's dependencies
359 precede it), with an unspecified (but deterministic) approach to
360 breaking ties.
362 Parameters
363 ----------
364 elements : iterable of `DimensionElement`.
365 Elements to be sorted.
366 reverse : `bool`, optional
367 If `True`, sort in the opposite order.
369 Returns
370 -------
371 sorted : `list` of `DimensionElement`
372 A sorted list containing the same elements that were given.
373 """
374 s = set(elements)
375 result = [element for element in self._elements if element in s or element.name in s]
376 if reverse:
377 result.reverse()
378 # mypy thinks this can return DimensionElements even if all the user
379 # passed it was Dimensions; we know better.
380 return result # type: ignore
382 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
383 """Make a dimension packer.
385 Constructs a `DimensionPacker` that can pack data ID dictionaries
386 into unique integers.
388 Parameters
389 ----------
390 name : `str`
391 Name of the packer, matching a key in the "packers" section of the
392 dimension configuration.
393 dataId : `DataCoordinate`
394 Fully-expanded data ID that identfies the at least the "fixed"
395 dimensions of the packer (i.e. those that are assumed/given,
396 setting the space over which packed integer IDs are unique).
397 ``dataId.hasRecords()`` must return `True`.
398 """
399 return self._packers[name](self, dataId)
401 def getEncodeLength(self) -> int:
402 """Return encoded size of graph.
404 Returns the size (in bytes) of the encoded size of `DimensionGraph`
405 instances in this universe.
407 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
408 information.
409 """
410 return math.ceil(len(self._dimensions)/8)
412 @classmethod
413 def _unpickle(cls, version: int) -> DimensionUniverse:
414 """Return an unpickled dimension universe.
416 Callable used for unpickling.
418 For internal use only.
419 """
420 try:
421 return cls._instances[version]
422 except KeyError as err:
423 raise pickle.UnpicklingError(
424 f"DimensionUniverse with version '{version}' "
425 f"not found. Note that DimensionUniverse objects are not "
426 f"truly serialized; when using pickle to transfer them "
427 f"between processes, an equivalent instance with the same "
428 f"version must already exist in the receiving process."
429 ) from err
431 def __reduce__(self) -> tuple:
432 return (self._unpickle, (self._version,))
434 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
435 # DimensionUniverse is recursively immutable; see note in @immutable
436 # decorator.
437 return self
439 # Class attributes below are shadowed by instance attributes, and are
440 # present just to hold the docstrings for those instance attributes.
442 empty: DimensionGraph
443 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
444 """
446 commonSkyPix: SkyPixDimension
447 """The special skypix dimension that is used to relate all other spatial
448 dimensions in the `Registry` database (`SkyPixDimension`).
449 """
451 _cache: Dict[FrozenSet[str], DimensionGraph]
453 _dimensions: NamedValueAbstractSet[Dimension]
455 _elements: NamedValueAbstractSet[DimensionElement]
457 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
459 _dimensionIndices: Dict[str, int]
461 _elementIndices: Dict[str, int]
463 _packers: Dict[str, DimensionPackerFactory]
465 _version: int