Coverage for python/lsst/daf/butler/core/dimensions/_universe.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 ClassVar,
30 Dict,
31 FrozenSet,
32 Iterable,
33 List,
34 Mapping,
35 Optional,
36 Set,
37 TYPE_CHECKING,
38 TypeVar,
39 Union,
40)
42from ..config import Config
43from ..named import NamedValueAbstractSet, NamedValueSet
44from .._topology import TopologicalSpace, TopologicalFamily
45from ..utils import cached_getter, immutable
46from ._config import DimensionConfig
47from ._elements import Dimension, DimensionElement
48from ._graph import DimensionGraph
49from ._governor import GovernorDimension
50from ._database import DatabaseDimensionElement
51from ._skypix import SkyPixDimension, SkyPixSystem
53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true
54 from ._coordinate import DataCoordinate
55 from ._packer import DimensionPacker, DimensionPackerFactory
56 from .construction import DimensionConstructionBuilder
59E = TypeVar("E", bound=DimensionElement)
62@immutable
63class DimensionUniverse:
64 """A parent class that represents a complete, self-consistent set of
65 dimensions and their relationships.
67 `DimensionUniverse` is not a class-level singleton, but all instances are
68 tracked in a singleton map keyed by the version number in the configuration
69 they were loaded from. Because these universes are solely responsible for
70 constructing `DimensionElement` instances, these are also indirectly
71 tracked by that singleton as well.
73 Parameters
74 ----------
75 config : `Config`, optional
76 Configuration object from which dimension definitions can be extracted.
77 Ignored if ``builder`` is provided, or if ``version`` is provided and
78 an instance with that version already exists.
79 version : `int`, optional
80 Integer version for this `DimensionUniverse`. If not provided, a
81 version will be obtained from ``builder`` or ``config``.
82 builder : `DimensionConstructionBuilder`, optional
83 Builder object used to initialize a new instance. Ignored if
84 ``version`` is provided and an instance with that version already
85 exists. Should not have had `~DimensionConstructionBuilder.finish`
86 called; this will be called if needed by `DimensionUniverse`.
87 """
89 _instances: ClassVar[Dict[int, DimensionUniverse]] = {}
90 """Singleton dictionary of all instances, keyed by version.
92 For internal use only.
93 """
95 def __new__(
96 cls,
97 config: Optional[Config] = None, *,
98 version: Optional[int] = None,
99 builder: Optional[DimensionConstructionBuilder] = None,
100 ) -> DimensionUniverse:
101 # Try to get a version first, to look for existing instances; try to
102 # do as little work as possible at this stage.
103 if version is None:
104 if builder is None:
105 config = DimensionConfig(config)
106 version = config["version"]
107 else:
108 version = builder.version
110 # See if an equivalent instance already exists.
111 self: Optional[DimensionUniverse] = cls._instances.get(version)
112 if self is not None:
113 return self
115 # Ensure we have a builder, building one from config if necessary.
116 if builder is None:
117 config = DimensionConfig(config)
118 builder = config.makeBuilder()
120 # Delegate to the builder for most of the construction work.
121 builder.finish()
123 # Create the universe instance and create core attributes, mostly
124 # copying from builder.
125 self = object.__new__(cls)
126 assert self is not None
127 self._cache = {}
128 self._dimensions = builder.dimensions
129 self._elements = builder.elements
130 self._topology = builder.topology
131 self._packers = builder.packers
132 self.commonSkyPix = self._dimensions[builder.commonSkyPixName]
134 # Attach self to all elements.
135 for element in self._elements:
136 element.universe = self
138 # Add attribute for special subsets of the graph.
139 self.empty = DimensionGraph(self, (), conform=False)
141 # Use the version number from the config as a key in the singleton
142 # dict containing all instances; that will let us transfer dimension
143 # objects between processes using pickle without actually going
144 # through real initialization, as long as a universe with the same
145 # version has already been constructed in the receiving process.
146 self._version = version
147 cls._instances[self._version] = self
149 # Build mappings from element to index. These are used for
150 # topological-sort comparison operators in DimensionElement itself.
151 self._elementIndices = {
152 name: i for i, name in enumerate(self._elements.names)
153 }
154 # Same for dimension to index, sorted topologically across required
155 # and implied. This is used for encode/decode.
156 self._dimensionIndices = {
157 name: i for i, name in enumerate(self._dimensions.names)
158 }
160 return self
162 def __repr__(self) -> str:
163 return f"DimensionUniverse({self._version})"
165 def __getitem__(self, name: str) -> DimensionElement:
166 return self._elements[name]
168 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
169 """Return the `DimensionElement` with the given name or a default.
171 Parameters
172 ----------
173 name : `str`
174 Name of the element.
175 default : `DimensionElement`, optional
176 Element to return if the named one does not exist. Defaults to
177 `None`.
179 Returns
180 -------
181 element : `DimensionElement`
182 The named element.
183 """
184 return self._elements.get(name, default)
186 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
187 """Return a set of all static elements in this universe.
189 Non-static elements that are created as needed may also exist, but
190 these are guaranteed to have no direct relationships to other elements
191 (though they may have spatial or temporal relationships).
193 Returns
194 -------
195 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
196 A frozen set of `DimensionElement` instances.
197 """
198 return self._elements
200 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
201 """Return a set of all static dimensions in this universe.
203 Non-static dimensions that are created as needed may also exist, but
204 these are guaranteed to have no direct relationships to other elements
205 (though they may have spatial or temporal relationships).
207 Returns
208 -------
209 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
210 A frozen set of `Dimension` instances.
211 """
212 return self._dimensions
214 @cached_getter
215 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
216 """Return a set of all `GovernorDimension` instances in this universe.
218 Returns
219 -------
220 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
221 A frozen set of `GovernorDimension` instances.
222 """
223 return NamedValueSet(
224 d for d in self._dimensions if isinstance(d, GovernorDimension)
225 ).freeze()
227 @cached_getter
228 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
229 """Return a set of all `DatabaseDimensionElement` instances in this
230 universe.
232 This does not include `GovernorDimension` instances, which are backed
233 by the database but do not inherit from `DatabaseDimensionElement`.
235 Returns
236 -------
237 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
238 A frozen set of `DatabaseDimensionElement` instances.
239 """
240 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
242 @property # type: ignore
243 @cached_getter
244 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
245 """All skypix systems known to this universe
246 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
247 """
248 return NamedValueSet([family for family in self._topology[TopologicalSpace.SPATIAL]
249 if isinstance(family, SkyPixSystem)]).freeze()
251 def getElementIndex(self, name: str) -> int:
252 """Return the position of the named dimension element in this
253 universe's sorting of all elements.
255 Parameters
256 ----------
257 name : `str`
258 Name of the element.
260 Returns
261 -------
262 index : `int`
263 Sorting index for this element.
264 """
265 return self._elementIndices[name]
267 def getDimensionIndex(self, name: str) -> int:
268 """Return the position of the named dimension in this universe's
269 sorting of all dimensions.
271 Parameters
272 ----------
273 name : `str`
274 Name of the dimension.
276 Returns
277 -------
278 index : `int`
279 Sorting index for this dimension.
281 Notes
282 -----
283 The dimension sort order for a universe is consistent with the element
284 order (all dimensions are elements), and either can be used to sort
285 dimensions if used consistently. But there are also some contexts in
286 which contiguous dimension-only indices are necessary or at least
287 desirable.
288 """
289 return self._dimensionIndices[name]
291 def expandDimensionNameSet(self, names: Set[str]) -> None:
292 """Expand a set of dimension names in-place to include recursive
293 dependencies.
295 This is an advanced interface for cases where constructing a
296 `DimensionGraph` (which also expands required dependencies) is
297 impossible or undesirable.
299 Parameters
300 ----------
301 names : `set` [ `str` ]
302 A true `set` of dimension names, to be expanded in-place.
303 """
304 # Keep iterating until the set of names stops growing. This is not as
305 # efficient as it could be, but we work pretty hard cache
306 # DimensionGraph instances to keep actual construction rare, so that
307 # shouldn't matter.
308 oldSize = len(names)
309 while True:
310 # iterate over a temporary copy so we can modify the original
311 for name in tuple(names):
312 names.update(self._dimensions[name].required.names)
313 names.update(self._dimensions[name].implied.names)
314 if oldSize == len(names):
315 break
316 else:
317 oldSize = len(names)
319 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
320 """Construct a `DimensionGraph` from a possibly-heterogenous iterable
321 of `Dimension` instances and string names thereof.
323 Constructing `DimensionGraph` directly from names or dimension
324 instances is slightly more efficient when it is known in advance that
325 the iterable is not heterogenous.
327 Parameters
328 ----------
329 iterable: iterable of `Dimension` or `str`
330 Dimensions that must be included in the returned graph (their
331 dependencies will be as well).
333 Returns
334 -------
335 graph : `DimensionGraph`
336 A `DimensionGraph` instance containing all given dimensions.
337 """
338 names = set()
339 for item in iterable:
340 try:
341 names.add(item.name) # type: ignore
342 except AttributeError:
343 names.add(item)
344 return DimensionGraph(universe=self, names=names)
346 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
347 """Return a sorted version of the given iterable of dimension elements.
349 The universe's sort order is topological (an element's dependencies
350 precede it), with an unspecified (but deterministic) approach to
351 breaking ties.
353 Parameters
354 ----------
355 elements : iterable of `DimensionElement`.
356 Elements to be sorted.
357 reverse : `bool`, optional
358 If `True`, sort in the opposite order.
360 Returns
361 -------
362 sorted : `list` of `DimensionElement`
363 A sorted list containing the same elements that were given.
364 """
365 s = set(elements)
366 result = [element for element in self._elements if element in s or element.name in s]
367 if reverse:
368 result.reverse()
369 # mypy thinks this can return DimensionElements even if all the user
370 # passed it was Dimensions; we know better.
371 return result # type: ignore
373 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
374 """Construct a `DimensionPacker` that can pack data ID dictionaries
375 into unique integers.
377 Parameters
378 ----------
379 name : `str`
380 Name of the packer, matching a key in the "packers" section of the
381 dimension configuration.
382 dataId : `DataCoordinate`
383 Fully-expanded data ID that identfies the at least the "fixed"
384 dimensions of the packer (i.e. those that are assumed/given,
385 setting the space over which packed integer IDs are unique).
386 ``dataId.hasRecords()`` must return `True`.
387 """
388 return self._packers[name](self, dataId)
390 def getEncodeLength(self) -> int:
391 """Return the size (in bytes) of the encoded size of `DimensionGraph`
392 instances in this universe.
394 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
395 information.
396 """
397 return math.ceil(len(self._dimensions)/8)
399 @classmethod
400 def _unpickle(cls, version: int) -> DimensionUniverse:
401 """Callable used for unpickling.
403 For internal use only.
404 """
405 try:
406 return cls._instances[version]
407 except KeyError as err:
408 raise pickle.UnpicklingError(
409 f"DimensionUniverse with version '{version}' "
410 f"not found. Note that DimensionUniverse objects are not "
411 f"truly serialized; when using pickle to transfer them "
412 f"between processes, an equivalent instance with the same "
413 f"version must already exist in the receiving process."
414 ) from err
416 def __reduce__(self) -> tuple:
417 return (self._unpickle, (self._version,))
419 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
420 # DimensionUniverse is recursively immutable; see note in @immutable
421 # decorator.
422 return self
424 # Class attributes below are shadowed by instance attributes, and are
425 # present just to hold the docstrings for those instance attributes.
427 empty: DimensionGraph
428 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
429 """
431 commonSkyPix: SkyPixDimension
432 """The special skypix dimension that is used to relate all other spatial
433 dimensions in the `Registry` database (`SkyPixDimension`).
434 """
436 _cache: Dict[FrozenSet[str], DimensionGraph]
438 _dimensions: NamedValueAbstractSet[Dimension]
440 _elements: NamedValueAbstractSet[DimensionElement]
442 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
444 _dimensionIndices: Dict[str, int]
446 _elementIndices: Dict[str, int]
448 _packers: Dict[str, DimensionPackerFactory]
450 _version: int