Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 36%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 TYPE_CHECKING,
30 ClassVar,
31 Dict,
32 FrozenSet,
33 Iterable,
34 List,
35 Mapping,
36 Optional,
37 Set,
38 TypeVar,
39 Union,
40)
42from lsst.utils.classes import cached_getter, immutable
44from .._topology import TopologicalFamily, TopologicalSpace
45from ..config import Config
46from ..named import NamedValueAbstractSet, NamedValueSet
47from ._config import DimensionConfig
48from ._database import DatabaseDimensionElement
49from ._elements import Dimension, DimensionElement
50from ._governor import GovernorDimension
51from ._graph import DimensionGraph
52from ._skypix import SkyPixDimension, SkyPixSystem
54if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 54 ↛ 55line 54 didn't jump to line 55, because the condition on line 54 was never true
55 from ._coordinate import DataCoordinate
56 from ._packer import DimensionPacker, DimensionPackerFactory
57 from .construction import DimensionConstructionBuilder
60E = TypeVar("E", bound=DimensionElement)
63@immutable
64class DimensionUniverse:
65 """Self-consistent set of dimensions.
67 A parent class that represents a complete, self-consistent set of
68 dimensions and their relationships.
70 `DimensionUniverse` is not a class-level singleton, but all instances are
71 tracked in a singleton map keyed by the version number in the configuration
72 they were loaded from. Because these universes are solely responsible for
73 constructing `DimensionElement` instances, these are also indirectly
74 tracked by that singleton as well.
76 Parameters
77 ----------
78 config : `Config`, optional
79 Configuration object from which dimension definitions can be extracted.
80 Ignored if ``builder`` is provided, or if ``version`` is provided and
81 an instance with that version already exists.
82 version : `int`, optional
83 Integer version for this `DimensionUniverse`. If not provided, a
84 version will be obtained from ``builder`` or ``config``.
85 builder : `DimensionConstructionBuilder`, optional
86 Builder object used to initialize a new instance. Ignored if
87 ``version`` is provided and an instance with that version already
88 exists. Should not have had `~DimensionConstructionBuilder.finish`
89 called; this will be called if needed by `DimensionUniverse`.
90 """
92 _instances: ClassVar[Dict[int, DimensionUniverse]] = {}
93 """Singleton dictionary of all instances, keyed by version.
95 For internal use only.
96 """
98 def __new__(
99 cls,
100 config: Optional[Config] = None,
101 *,
102 version: Optional[int] = None,
103 builder: Optional[DimensionConstructionBuilder] = None,
104 ) -> DimensionUniverse:
105 # Try to get a version first, to look for existing instances; try to
106 # do as little work as possible at this stage.
107 if version is None:
108 if builder is None:
109 config = DimensionConfig(config)
110 version = config["version"]
111 else:
112 version = builder.version
114 # See if an equivalent instance already exists.
115 self: Optional[DimensionUniverse] = cls._instances.get(version)
116 if self is not None:
117 return self
119 # Ensure we have a builder, building one from config if necessary.
120 if builder is None:
121 config = DimensionConfig(config)
122 builder = config.makeBuilder()
124 # Delegate to the builder for most of the construction work.
125 builder.finish()
127 # Create the universe instance and create core attributes, mostly
128 # copying from builder.
129 self = object.__new__(cls)
130 assert self is not None
131 self._cache = {}
132 self._dimensions = builder.dimensions
133 self._elements = builder.elements
134 self._topology = builder.topology
135 self._packers = builder.packers
136 self.dimensionConfig = builder.config
137 commonSkyPix = self._dimensions[builder.commonSkyPixName]
138 assert isinstance(commonSkyPix, SkyPixDimension)
139 self.commonSkyPix = commonSkyPix
141 # Attach self to all elements.
142 for element in self._elements:
143 element.universe = self
145 # Add attribute for special subsets of the graph.
146 self.empty = DimensionGraph(self, (), conform=False)
148 # Use the version number from the config as a key in the singleton
149 # dict containing all instances; that will let us transfer dimension
150 # objects between processes using pickle without actually going
151 # through real initialization, as long as a universe with the same
152 # version has already been constructed in the receiving process.
153 self._version = version
154 cls._instances[self._version] = self
156 # Build mappings from element to index. These are used for
157 # topological-sort comparison operators in DimensionElement itself.
158 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
159 # Same for dimension to index, sorted topologically across required
160 # and implied. This is used for encode/decode.
161 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
163 return self
165 def __repr__(self) -> str:
166 return f"DimensionUniverse({self._version})"
168 def __getitem__(self, name: str) -> DimensionElement:
169 return self._elements[name]
171 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
172 """Return the `DimensionElement` with the given name or a default.
174 Parameters
175 ----------
176 name : `str`
177 Name of the element.
178 default : `DimensionElement`, optional
179 Element to return if the named one does not exist. Defaults to
180 `None`.
182 Returns
183 -------
184 element : `DimensionElement`
185 The named element.
186 """
187 return self._elements.get(name, default)
189 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
190 """Return a set of all static elements in this universe.
192 Non-static elements that are created as needed may also exist, but
193 these are guaranteed to have no direct relationships to other elements
194 (though they may have spatial or temporal relationships).
196 Returns
197 -------
198 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
199 A frozen set of `DimensionElement` instances.
200 """
201 return self._elements
203 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
204 """Return a set of all static dimensions in this universe.
206 Non-static dimensions that are created as needed may also exist, but
207 these are guaranteed to have no direct relationships to other elements
208 (though they may have spatial or temporal relationships).
210 Returns
211 -------
212 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
213 A frozen set of `Dimension` instances.
214 """
215 return self._dimensions
217 @cached_getter
218 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
219 """Return a set of all `GovernorDimension` instances in this universe.
221 Returns
222 -------
223 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
224 A frozen set of `GovernorDimension` instances.
225 """
226 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
228 @cached_getter
229 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
230 """Return set of all `DatabaseDimensionElement` instances in universe.
232 This does not include `GovernorDimension` instances, which are backed
233 by the database but do not inherit from `DatabaseDimensionElement`.
235 Returns
236 -------
237 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
238 A frozen set of `DatabaseDimensionElement` instances.
239 """
240 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
242 @property # type: ignore
243 @cached_getter
244 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
245 """All skypix systems known to this universe.
247 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
248 """
249 return NamedValueSet(
250 [
251 family
252 for family in self._topology[TopologicalSpace.SPATIAL]
253 if isinstance(family, SkyPixSystem)
254 ]
255 ).freeze()
257 def getElementIndex(self, name: str) -> int:
258 """Return the position of the named dimension element.
260 The position is in this universe's sorting of all elements.
262 Parameters
263 ----------
264 name : `str`
265 Name of the element.
267 Returns
268 -------
269 index : `int`
270 Sorting index for this element.
271 """
272 return self._elementIndices[name]
274 def getDimensionIndex(self, name: str) -> int:
275 """Return the position of the named dimension.
277 This position is in this universe's sorting of all dimensions.
279 Parameters
280 ----------
281 name : `str`
282 Name of the dimension.
284 Returns
285 -------
286 index : `int`
287 Sorting index for this dimension.
289 Notes
290 -----
291 The dimension sort order for a universe is consistent with the element
292 order (all dimensions are elements), and either can be used to sort
293 dimensions if used consistently. But there are also some contexts in
294 which contiguous dimension-only indices are necessary or at least
295 desirable.
296 """
297 return self._dimensionIndices[name]
299 def expandDimensionNameSet(self, names: Set[str]) -> None:
300 """Expand a set of dimension names in-place.
302 Includes recursive dependencies.
304 This is an advanced interface for cases where constructing a
305 `DimensionGraph` (which also expands required dependencies) is
306 impossible or undesirable.
308 Parameters
309 ----------
310 names : `set` [ `str` ]
311 A true `set` of dimension names, to be expanded in-place.
312 """
313 # Keep iterating until the set of names stops growing. This is not as
314 # efficient as it could be, but we work pretty hard cache
315 # DimensionGraph instances to keep actual construction rare, so that
316 # shouldn't matter.
317 oldSize = len(names)
318 while True:
319 # iterate over a temporary copy so we can modify the original
320 for name in tuple(names):
321 names.update(self._dimensions[name].required.names)
322 names.update(self._dimensions[name].implied.names)
323 if oldSize == len(names):
324 break
325 else:
326 oldSize = len(names)
328 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
329 """Construct graph from iterable.
331 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
332 of `Dimension` instances and string names thereof.
334 Constructing `DimensionGraph` directly from names or dimension
335 instances is slightly more efficient when it is known in advance that
336 the iterable is not heterogenous.
338 Parameters
339 ----------
340 iterable: iterable of `Dimension` or `str`
341 Dimensions that must be included in the returned graph (their
342 dependencies will be as well).
344 Returns
345 -------
346 graph : `DimensionGraph`
347 A `DimensionGraph` instance containing all given dimensions.
348 """
349 names = set()
350 for item in iterable:
351 try:
352 names.add(item.name) # type: ignore
353 except AttributeError:
354 names.add(item)
355 return DimensionGraph(universe=self, names=names)
357 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
358 """Return a sorted version of the given iterable of dimension elements.
360 The universe's sort order is topological (an element's dependencies
361 precede it), with an unspecified (but deterministic) approach to
362 breaking ties.
364 Parameters
365 ----------
366 elements : iterable of `DimensionElement`.
367 Elements to be sorted.
368 reverse : `bool`, optional
369 If `True`, sort in the opposite order.
371 Returns
372 -------
373 sorted : `list` of `DimensionElement`
374 A sorted list containing the same elements that were given.
375 """
376 s = set(elements)
377 result = [element for element in self._elements if element in s or element.name in s]
378 if reverse:
379 result.reverse()
380 # mypy thinks this can return DimensionElements even if all the user
381 # passed it was Dimensions; we know better.
382 return result # type: ignore
384 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
385 """Make a dimension packer.
387 Constructs a `DimensionPacker` that can pack data ID dictionaries
388 into unique integers.
390 Parameters
391 ----------
392 name : `str`
393 Name of the packer, matching a key in the "packers" section of the
394 dimension configuration.
395 dataId : `DataCoordinate`
396 Fully-expanded data ID that identifies the at least the "fixed"
397 dimensions of the packer (i.e. those that are assumed/given,
398 setting the space over which packed integer IDs are unique).
399 ``dataId.hasRecords()`` must return `True`.
400 """
401 return self._packers[name](self, dataId)
403 def getEncodeLength(self) -> int:
404 """Return encoded size of graph.
406 Returns the size (in bytes) of the encoded size of `DimensionGraph`
407 instances in this universe.
409 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
410 information.
411 """
412 return math.ceil(len(self._dimensions) / 8)
414 @classmethod
415 def _unpickle(cls, version: int) -> DimensionUniverse:
416 """Return an unpickled dimension universe.
418 Callable used for unpickling.
420 For internal use only.
421 """
422 try:
423 return cls._instances[version]
424 except KeyError as err:
425 raise pickle.UnpicklingError(
426 f"DimensionUniverse with version '{version}' "
427 f"not found. Note that DimensionUniverse objects are not "
428 f"truly serialized; when using pickle to transfer them "
429 f"between processes, an equivalent instance with the same "
430 f"version must already exist in the receiving process."
431 ) from err
433 def __reduce__(self) -> tuple:
434 return (self._unpickle, (self._version,))
436 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
437 # DimensionUniverse is recursively immutable; see note in @immutable
438 # decorator.
439 return self
441 # Class attributes below are shadowed by instance attributes, and are
442 # present just to hold the docstrings for those instance attributes.
444 empty: DimensionGraph
445 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
446 """
448 commonSkyPix: SkyPixDimension
449 """The special skypix dimension that is used to relate all other spatial
450 dimensions in the `Registry` database (`SkyPixDimension`).
451 """
453 dimensionConfig: DimensionConfig
454 """The configuration used to create this Universe (`DimensionConfig`)."""
456 _cache: Dict[FrozenSet[str], DimensionGraph]
458 _dimensions: NamedValueAbstractSet[Dimension]
460 _elements: NamedValueAbstractSet[DimensionElement]
462 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
464 _dimensionIndices: Dict[str, int]
466 _elementIndices: Dict[str, int]
468 _packers: Dict[str, DimensionPackerFactory]
470 _version: int