Coverage for python/lsst/daf/butler/core/dimensions/universe.py : 33%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 ClassVar,
30 Dict,
31 FrozenSet,
32 Iterable,
33 List,
34 Optional,
35 TYPE_CHECKING,
36 TypeVar,
37 Union,
38)
40from ..config import Config
41from ..named import NamedValueSet
42from ..utils import immutable
43from .elements import Dimension, DimensionElement, SkyPixDimension
44from .graph import DimensionGraph
45from .config import processElementsConfig, processSkyPixConfig, DimensionConfig
46from .packer import DimensionPackerFactory
48if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true
49 from .coordinate import DataCoordinate
50 from .packer import DimensionPacker
53E = TypeVar("E", bound=DimensionElement)
56@immutable
57class DimensionUniverse:
58 """A parent class that represents a complete, self-consistent set of
59 dimensions and their relationships.
61 `DimensionUniverse` is not a class-level singleton, but all instances are
62 tracked in a singleton map keyed by the version number in the configuration
63 they were loaded from. Because these universes are solely responsible for
64 constructing `DimensionElement` instances, these are also indirectly
65 tracked by that singleton as well.
67 Parameters
68 ----------
69 config : `Config`, optional
70 Configuration describing the dimensions and their relationships. If
71 not provided, default configuration (from ``dimensions.yaml`` in
72 this package's ``configs`` resource directory) will be loaded.
73 """
75 _instances: ClassVar[Dict[int, DimensionUniverse]] = {}
76 """Singleton dictionary of all instances, keyed by version.
78 For internal use only.
79 """
81 def __new__(cls, config: Optional[Config] = None) -> DimensionUniverse:
82 # Normalize the config and apply defaults.
83 config = DimensionConfig(config)
85 # First see if an equivalent instance already exists.
86 version = config["version"]
87 self: Optional[DimensionUniverse] = cls._instances.get(version)
88 if self is not None:
89 return self
91 # Create the universe instance and add core attributes.
92 self = object.__new__(cls)
93 assert self is not None
94 self._cache = {}
95 self._dimensions = NamedValueSet()
96 self._elements = NamedValueSet()
98 # Read the skypix dimensions from config.
99 skyPixDimensions, self.commonSkyPix = processSkyPixConfig(config["skypix"])
100 # Add the skypix dimensions to the universe after sorting
101 # lexicographically (no topological sort because skypix dimensions
102 # never have any dependencies).
103 for name in sorted(skyPixDimensions):
104 skyPixDimensions[name]._finish(self, {})
106 # Read the other dimension elements from config.
107 elementsToDo = processElementsConfig(config["elements"])
108 # Add elements to the universe in topological order by identifying at
109 # each outer iteration which elements have already had all of their
110 # dependencies added.
111 while elementsToDo:
112 unblocked = [name for name, element in elementsToDo.items()
113 if element._related.dependencies.isdisjoint(elementsToDo.keys())]
114 unblocked.sort() # Break ties lexicographically.
115 if not unblocked:
116 raise RuntimeError(f"Cycle detected in dimension elements: {elementsToDo.keys()}.")
117 for name in unblocked:
118 # Finish initialization of the element with steps that
119 # depend on those steps already having been run for all
120 # dependencies.
121 # This includes adding the element to self.elements and
122 # (if appropriate) self.dimensions.
123 elementsToDo.pop(name)._finish(self, elementsToDo)
125 # Add attributes for special subsets of the graph.
126 self.empty = DimensionGraph(self, (), conform=False)
128 # Set up factories for dataId packers as defined by config.
129 # MyPy is totally confused by us setting attributes on self in __new__.
130 self._packers = {}
131 for name, subconfig in config.get("packers", {}).items():
132 self._packers[name] = DimensionPackerFactory.fromConfig(universe=self, # type: ignore
133 config=subconfig)
135 # Use the version number from the config as a key in the singleton
136 # dict containing all instances; that will let us transfer dimension
137 # objects between processes using pickle without actually going
138 # through real initialization, as long as a universe with the same
139 # version has already been constructed in the receiving process.
140 self._version = version
141 cls._instances[self._version] = self
143 # Build mappings from element to index. These are used for
144 # topological-sort comparison operators in DimensionElement itself.
145 self._elementIndices = {
146 name: i for i, name in enumerate(self._elements.names)
147 }
148 # Same for dimension to index, sorted topologically across required
149 # and implied. This is used for encode/decode.
150 self._dimensionIndices = {
151 name: i for i, name in enumerate(self._dimensions.names)
152 }
154 # Freeze internal sets so we can return them in methods without
155 # worrying about modifications.
156 self._elements.freeze()
157 self._dimensions.freeze()
158 return self
160 def __repr__(self) -> str:
161 return f"DimensionUniverse({self._version})"
163 def __getitem__(self, name: str) -> DimensionElement:
164 return self._elements[name]
166 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]:
167 """Return the `DimensionElement` with the given name or a default.
169 Parameters
170 ----------
171 name : `str`
172 Name of the element.
173 default : `DimensionElement`, optional
174 Element to return if the named one does not exist. Defaults to
175 `None`.
177 Returns
178 -------
179 element : `DimensionElement`
180 The named element.
181 """
182 return self._elements.get(name, default)
184 def getStaticElements(self) -> NamedValueSet[DimensionElement]:
185 """Return a set of all static elements in this universe.
187 Non-static elements that are created as needed may also exist, but
188 these are guaranteed to have no direct relationships to other elements
189 (though they may have spatial or temporal relationships).
191 Returns
192 -------
193 elements : `NamedValueSet` [ `DimensionElement` ]
194 A frozen set of `DimensionElement` instances.
195 """
196 return self._elements
198 def getStaticDimensions(self) -> NamedValueSet[Dimension]:
199 """Return a set of all static dimensions in this universe.
201 Non-static dimensions that are created as needed may also exist, but
202 these are guaranteed to have no direct relationships to other elements
203 (though they may have spatial or temporal relationships).
205 Returns
206 -------
207 dimensions : `NamedValueSet` [ `Dimension` ]
208 A frozen set of `Dimension` instances.
209 """
210 return self._dimensions
212 def getElementIndex(self, name: str) -> int:
213 """Return the position of the named dimension element in this
214 universe's sorting of all elements.
216 Parameters
217 ----------
218 name : `str`
219 Name of the element.
221 Returns
222 -------
223 index : `int`
224 Sorting index for this element.
225 """
226 return self._elementIndices[name]
228 def getDimensionIndex(self, name: str) -> int:
229 """Return the position of the named dimension in this universe's
230 sorting of all dimensions.
232 Parameters
233 ----------
234 name : `str`
235 Name of the dimension.
237 Returns
238 -------
239 index : `int`
240 Sorting index for this dimension.
242 Notes
243 -----
244 The dimension sort order for a universe is consistent with the element
245 order (all dimensions are elements), and either can be used to sort
246 dimensions if used consistently. But there are also some contexts in
247 which contiguous dimension-only indices are necessary or at least
248 desirable.
249 """
250 return self._dimensionIndices[name]
252 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
253 """Construct a `DimensionGraph` from a possibly-heterogenous iterable
254 of `Dimension` instances and string names thereof.
256 Constructing `DimensionGraph` directly from names or dimension
257 instances is slightly more efficient when it is known in advance that
258 the iterable is not heterogenous.
260 Parameters
261 ----------
262 iterable: iterable of `Dimension` or `str`
263 Dimensions that must be included in the returned graph (their
264 dependencies will be as well).
266 Returns
267 -------
268 graph : `DimensionGraph`
269 A `DimensionGraph` instance containing all given dimensions.
270 """
271 names = set()
272 for item in iterable:
273 try:
274 names.add(item.name) # type: ignore
275 except AttributeError:
276 names.add(item)
277 return DimensionGraph(universe=self, names=names)
279 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
280 """Return a sorted version of the given iterable of dimension elements.
282 The universe's sort order is topological (an element's dependencies
283 precede it), with an unspecified (but deterministic) approach to
284 breaking ties.
286 Parameters
287 ----------
288 elements : iterable of `DimensionElement`.
289 Elements to be sorted.
290 reverse : `bool`, optional
291 If `True`, sort in the opposite order.
293 Returns
294 -------
295 sorted : `list` of `DimensionElement`
296 A sorted list containing the same elements that were given.
297 """
298 s = set(elements)
299 result = [element for element in self._elements if element in s or element.name in s]
300 if reverse:
301 result.reverse()
302 # mypy thinks this can return DimensionElements even if all the user
303 # passed it was Dimensions; we know better.
304 return result # type: ignore
306 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
307 """Construct a `DimensionPacker` that can pack data ID dictionaries
308 into unique integers.
310 Parameters
311 ----------
312 name : `str`
313 Name of the packer, matching a key in the "packers" section of the
314 dimension configuration.
315 dataId : `DataCoordinate`
316 Fully-expanded data ID that identfies the at least the "fixed"
317 dimensions of the packer (i.e. those that are assumed/given,
318 setting the space over which packed integer IDs are unique).
319 ``dataId.hasRecords()`` must return `True`.
320 """
321 return self._packers[name](dataId)
323 def getEncodeLength(self) -> int:
324 """Return the size (in bytes) of the encoded size of `DimensionGraph`
325 instances in this universe.
327 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
328 information.
329 """
330 return math.ceil(len(self._dimensions)/8)
332 @classmethod
333 def _unpickle(cls, version: int) -> DimensionUniverse:
334 """Callable used for unpickling.
336 For internal use only.
337 """
338 try:
339 return cls._instances[version]
340 except KeyError as err:
341 raise pickle.UnpicklingError(
342 f"DimensionUniverse with version '{version}' "
343 f"not found. Note that DimensionUniverse objects are not "
344 f"truly serialized; when using pickle to transfer them "
345 f"between processes, an equivalent instance with the same "
346 f"version must already exist in the receiving process."
347 ) from err
349 def __reduce__(self) -> tuple:
350 return (self._unpickle, (self._version,))
352 # Class attributes below are shadowed by instance attributes, and are
353 # present just to hold the docstrings for those instance attributes.
355 empty: DimensionGraph
356 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
357 """
359 commonSkyPix: SkyPixDimension
360 """The special skypix dimension that is used to relate all other spatial
361 dimensions in the `Registry` database (`SkyPixDimension`).
362 """
364 _cache: Dict[FrozenSet[str], DimensionGraph]
366 _dimensions: NamedValueSet[Dimension]
368 _elements: NamedValueSet[DimensionElement]
370 _dimensionIndices: Dict[str, int]
372 _elementIndices: Dict[str, int]
374 _packers: Dict[str, DimensionPackerFactory]
376 _version: int