Coverage for python/lsst/daf/butler/core/dimensions/_universe.py : 36%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 ClassVar,
30 Dict,
31 FrozenSet,
32 Iterable,
33 List,
34 Mapping,
35 Optional,
36 Set,
37 TYPE_CHECKING,
38 TypeVar,
39 Union,
40)
42from ..config import Config
43from ..named import NamedValueAbstractSet, NamedValueSet
44from .._topology import TopologicalSpace, TopologicalFamily
45from ..utils import cached_getter, immutable
46from ._config import DimensionConfig
47from ._elements import Dimension, DimensionElement
48from ._graph import DimensionGraph
49from ._governor import GovernorDimension
50from ._database import DatabaseDimensionElement
51from ._skypix import SkyPixDimension, SkyPixSystem
53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true
54 from ._coordinate import DataCoordinate
55 from ._packer import DimensionPacker, DimensionPackerFactory
56 from .construction import DimensionConstructionBuilder
59E = TypeVar("E", bound=DimensionElement)
62@immutable
63class DimensionUniverse:
64 """A parent class that represents a complete, self-consistent set of
65 dimensions and their relationships.
67 `DimensionUniverse` is not a class-level singleton, but all instances are
68 tracked in a singleton map keyed by the version number in the configuration
69 they were loaded from. Because these universes are solely responsible for
70 constructing `DimensionElement` instances, these are also indirectly
71 tracked by that singleton as well.
73 Parameters
74 ----------
75 config : `Config`, optional
76 Configuration object from which dimension definitions can be extracted.
77 Ignored if ``builder`` is provided, or if ``version`` is provided and
78 an instance with that version already exists.
79 version : `int`, optional
80 Integer version for this `DimensionUniverse`. If not provided, a
81 version will be obtained from ``builder`` or ``config``.
82 builder : `DimensionConstructionBuilder`, optional
83 Builder object used to initialize a new instance. Ignored if
84 ``version`` is provided and an instance with that version already
85 exists. Should not have had `~DimensionConstructionBuilder.finish`
86 called; this will be called if needed by `DimensionUniverse`.
87 """
89 _instances: ClassVar[Dict[int, DimensionUniverse]] = {}
90 """Singleton dictionary of all instances, keyed by version.
92 For internal use only.
93 """
95 def __new__(
96 cls,
97 config: Optional[Config] = None, *,
98 version: Optional[int] = None,
99 builder: Optional[DimensionConstructionBuilder] = None,
100 ) -> DimensionUniverse:
101 # Try to get a version first, to look for existing instances; try to
102 # do as little work as possible at this stage.
103 if version is None:
104 if builder is None:
105 config = DimensionConfig(config)
106 version = config["version"]
107 else:
108 version = builder.version
110 # See if an equivalent instance already exists.
111 self: Optional[DimensionUniverse] = cls._instances.get(version)
112 if self is not None:
113 return self
115 # Ensure we have a builder, building one from config if necessary.
116 if builder is None:
117 config = DimensionConfig(config)
118 builder = config.makeBuilder()
120 # Delegate to the builder for most of the construction work.
121 builder.finish()
123 # Create the universe instance and create core attributes, mostly
124 # copying from builder.
125 self = object.__new__(cls)
126 assert self is not None
127 self._cache = {}
128 self._dimensions = builder.dimensions
129 self._elements = builder.elements
130 self._topology = builder.topology
131 self._packers = builder.packers
132 commonSkyPix = self._dimensions[builder.commonSkyPixName]
133 assert isinstance(commonSkyPix, SkyPixDimension)
134 self.commonSkyPix = commonSkyPix
136 # Attach self to all elements.
137 for element in self._elements:
138 element.universe = self
140 # Add attribute for special subsets of the graph.
141 self.empty = DimensionGraph(self, (), conform=False)
143 # Use the version number from the config as a key in the singleton
144 # dict containing all instances; that will let us transfer dimension
145 # objects between processes using pickle without actually going
146 # through real initialization, as long as a universe with the same
147 # version has already been constructed in the receiving process.
148 self._version = version
149 cls._instances[self._version] = self
151 # Build mappings from element to index. These are used for
152 # topological-sort comparison operators in DimensionElement itself.
153 self._elementIndices = {
154 name: i for i, name in enumerate(self._elements.names)
155 }
156 # Same for dimension to index, sorted topologically across required
157 # and implied. This is used for encode/decode.
158 self._dimensionIndices = {
159 name: i for i, name in enumerate(self._dimensions.names)
160 }
162 return self
164 def __repr__(self) -> str:
165 return f"DimensionUniverse({self._version})"
167 def __getitem__(self, name: str) -> DimensionElement:
168 return self._elements[name]
170 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
171 """Return the `DimensionElement` with the given name or a default.
173 Parameters
174 ----------
175 name : `str`
176 Name of the element.
177 default : `DimensionElement`, optional
178 Element to return if the named one does not exist. Defaults to
179 `None`.
181 Returns
182 -------
183 element : `DimensionElement`
184 The named element.
185 """
186 return self._elements.get(name, default)
188 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
189 """Return a set of all static elements in this universe.
191 Non-static elements that are created as needed may also exist, but
192 these are guaranteed to have no direct relationships to other elements
193 (though they may have spatial or temporal relationships).
195 Returns
196 -------
197 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
198 A frozen set of `DimensionElement` instances.
199 """
200 return self._elements
202 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
203 """Return a set of all static dimensions in this universe.
205 Non-static dimensions that are created as needed may also exist, but
206 these are guaranteed to have no direct relationships to other elements
207 (though they may have spatial or temporal relationships).
209 Returns
210 -------
211 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
212 A frozen set of `Dimension` instances.
213 """
214 return self._dimensions
216 @cached_getter
217 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
218 """Return a set of all `GovernorDimension` instances in this universe.
220 Returns
221 -------
222 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
223 A frozen set of `GovernorDimension` instances.
224 """
225 return NamedValueSet(
226 d for d in self._dimensions if isinstance(d, GovernorDimension)
227 ).freeze()
229 @cached_getter
230 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
231 """Return a set of all `DatabaseDimensionElement` instances in this
232 universe.
234 This does not include `GovernorDimension` instances, which are backed
235 by the database but do not inherit from `DatabaseDimensionElement`.
237 Returns
238 -------
239 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
240 A frozen set of `DatabaseDimensionElement` instances.
241 """
242 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
244 @property # type: ignore
245 @cached_getter
246 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
247 """All skypix systems known to this universe
248 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
249 """
250 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL]
251 if isinstance(family, SkyPixSystem)]).freeze()
253 def getElementIndex(self, name: str) -> int:
254 """Return the position of the named dimension element in this
255 universe's sorting of all elements.
257 Parameters
258 ----------
259 name : `str`
260 Name of the element.
262 Returns
263 -------
264 index : `int`
265 Sorting index for this element.
266 """
267 return self._elementIndices[name]
269 def getDimensionIndex(self, name: str) -> int:
270 """Return the position of the named dimension in this universe's
271 sorting of all dimensions.
273 Parameters
274 ----------
275 name : `str`
276 Name of the dimension.
278 Returns
279 -------
280 index : `int`
281 Sorting index for this dimension.
283 Notes
284 -----
285 The dimension sort order for a universe is consistent with the element
286 order (all dimensions are elements), and either can be used to sort
287 dimensions if used consistently. But there are also some contexts in
288 which contiguous dimension-only indices are necessary or at least
289 desirable.
290 """
291 return self._dimensionIndices[name]
293 def expandDimensionNameSet(self, names: Set[str]) -> None:
294 """Expand a set of dimension names in-place to include recursive
295 dependencies.
297 This is an advanced interface for cases where constructing a
298 `DimensionGraph` (which also expands required dependencies) is
299 impossible or undesirable.
301 Parameters
302 ----------
303 names : `set` [ `str` ]
304 A true `set` of dimension names, to be expanded in-place.
305 """
306 # Keep iterating until the set of names stops growing. This is not as
307 # efficient as it could be, but we work pretty hard cache
308 # DimensionGraph instances to keep actual construction rare, so that
309 # shouldn't matter.
310 oldSize = len(names)
311 while True:
312 # iterate over a temporary copy so we can modify the original
313 for name in tuple(names):
314 names.update(self._dimensions[name].required.names)
315 names.update(self._dimensions[name].implied.names)
316 if oldSize == len(names):
317 break
318 else:
319 oldSize = len(names)
321 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
322 """Construct a `DimensionGraph` from a possibly-heterogenous iterable
323 of `Dimension` instances and string names thereof.
325 Constructing `DimensionGraph` directly from names or dimension
326 instances is slightly more efficient when it is known in advance that
327 the iterable is not heterogenous.
329 Parameters
330 ----------
331 iterable: iterable of `Dimension` or `str`
332 Dimensions that must be included in the returned graph (their
333 dependencies will be as well).
335 Returns
336 -------
337 graph : `DimensionGraph`
338 A `DimensionGraph` instance containing all given dimensions.
339 """
340 names = set()
341 for item in iterable:
342 try:
343 names.add(item.name) # type: ignore
344 except AttributeError:
345 names.add(item)
346 return DimensionGraph(universe=self, names=names)
348 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
349 """Return a sorted version of the given iterable of dimension elements.
351 The universe's sort order is topological (an element's dependencies
352 precede it), with an unspecified (but deterministic) approach to
353 breaking ties.
355 Parameters
356 ----------
357 elements : iterable of `DimensionElement`.
358 Elements to be sorted.
359 reverse : `bool`, optional
360 If `True`, sort in the opposite order.
362 Returns
363 -------
364 sorted : `list` of `DimensionElement`
365 A sorted list containing the same elements that were given.
366 """
367 s = set(elements)
368 result = [element for element in self._elements if element in s or element.name in s]
369 if reverse:
370 result.reverse()
371 # mypy thinks this can return DimensionElements even if all the user
372 # passed it was Dimensions; we know better.
373 return result # type: ignore
375 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
376 """Construct a `DimensionPacker` that can pack data ID dictionaries
377 into unique integers.
379 Parameters
380 ----------
381 name : `str`
382 Name of the packer, matching a key in the "packers" section of the
383 dimension configuration.
384 dataId : `DataCoordinate`
385 Fully-expanded data ID that identfies the at least the "fixed"
386 dimensions of the packer (i.e. those that are assumed/given,
387 setting the space over which packed integer IDs are unique).
388 ``dataId.hasRecords()`` must return `True`.
389 """
390 return self._packers[name](self, dataId)
392 def getEncodeLength(self) -> int:
393 """Return the size (in bytes) of the encoded size of `DimensionGraph`
394 instances in this universe.
396 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
397 information.
398 """
399 return math.ceil(len(self._dimensions)/8)
401 @classmethod
402 def _unpickle(cls, version: int) -> DimensionUniverse:
403 """Callable used for unpickling.
405 For internal use only.
406 """
407 try:
408 return cls._instances[version]
409 except KeyError as err:
410 raise pickle.UnpicklingError(
411 f"DimensionUniverse with version '{version}' "
412 f"not found. Note that DimensionUniverse objects are not "
413 f"truly serialized; when using pickle to transfer them "
414 f"between processes, an equivalent instance with the same "
415 f"version must already exist in the receiving process."
416 ) from err
418 def __reduce__(self) -> tuple:
419 return (self._unpickle, (self._version,))
421 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
422 # DimensionUniverse is recursively immutable; see note in @immutable
423 # decorator.
424 return self
426 # Class attributes below are shadowed by instance attributes, and are
427 # present just to hold the docstrings for those instance attributes.
429 empty: DimensionGraph
430 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
431 """
433 commonSkyPix: SkyPixDimension
434 """The special skypix dimension that is used to relate all other spatial
435 dimensions in the `Registry` database (`SkyPixDimension`).
436 """
438 _cache: Dict[FrozenSet[str], DimensionGraph]
440 _dimensions: NamedValueAbstractSet[Dimension]
442 _elements: NamedValueAbstractSet[DimensionElement]
444 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
446 _dimensionIndices: Dict[str, int]
448 _elementIndices: Dict[str, int]
450 _packers: Dict[str, DimensionPackerFactory]
452 _version: int