Coverage for python/lsst/daf/butler/core/dimensions/universe.py : 31%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionUniverse"]
26import math
27import pickle
28from typing import (
29 ClassVar,
30 Dict,
31 FrozenSet,
32 Iterable,
33 List,
34 Optional,
35 TYPE_CHECKING,
36 TypeVar,
37 Union,
38)
40from ..config import Config
41from ..named import NamedValueSet
42from ..utils import immutable
43from .elements import Dimension, DimensionElement, SkyPixDimension
44from .graph import DimensionGraph
45from .config import processElementsConfig, processSkyPixConfig, DimensionConfig
46from .packer import DimensionPackerFactory
48if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true
49 from .coordinate import DataCoordinate
50 from .packer import DimensionPacker
53E = TypeVar("E", bound=DimensionElement)
56@immutable
57class DimensionUniverse(DimensionGraph):
58 """A special `DimensionGraph` that constructs and manages a complete set of
59 compatible dimensions.
61 `DimensionUniverse` is not a class-level singleton, but all instances are
62 tracked in a singleton map keyed by the version number in the configuration
63 they were loaded from. Because these universes are solely responsible for
64 constructing `DimensionElement` instances, these are also indirectly
65 tracked by that singleton as well.
67 Parameters
68 ----------
69 config : `Config`, optional
70 Configuration describing the dimensions and their relationships. If
71 not provided, default configuration (from
72 ``daf_butler/config/dimensions.yaml``) wil be loaded.
73 """
75 _instances: ClassVar[Dict[int, DimensionUniverse]] = {}
76 """Singleton dictionary of all instances, keyed by version.
78 For internal use only.
79 """
81 def __new__(cls, config: Optional[Config] = None) -> DimensionUniverse:
82 # Normalize the config and apply defaults.
83 config = DimensionConfig(config)
85 # First see if an equivalent instance already exists.
86 version = config["version"]
87 self: Optional[DimensionUniverse] = cls._instances.get(version)
88 if self is not None:
89 return self
91 # Create the universe instance and add core attributes.
92 # We don't want any of what DimensionGraph.__new__ does, so we just go
93 # straight to object.__new__. The C++ side of my brain is offended by
94 # this, but I think it's the right approach in Python, where we don't
95 # have the option of having multiple constructors with different roles.
96 self = object.__new__(cls)
97 self.universe = self
98 self._cache = {}
99 self.dimensions = NamedValueSet()
100 self.elements = NamedValueSet()
102 # Read the skypix dimensions from config.
103 skyPixDimensions, self.commonSkyPix = processSkyPixConfig(config["skypix"])
104 # Add the skypix dimensions to the universe after sorting
105 # lexicographically (no topological sort because skypix dimensions
106 # never have any dependencies).
107 for name in sorted(skyPixDimensions):
108 skyPixDimensions[name]._finish(self, {})
110 # Read the other dimension elements from config.
111 elementsToDo = processElementsConfig(config["elements"])
112 # Add elements to the universe in topological order by identifying at
113 # each outer iteration which elements have already had all of their
114 # dependencies added.
115 while elementsToDo:
116 unblocked = [name for name, element in elementsToDo.items()
117 if element._related.dependencies.isdisjoint(elementsToDo.keys())]
118 unblocked.sort() # Break ties lexicographically.
119 if not unblocked:
120 raise RuntimeError(f"Cycle detected in dimension elements: {elementsToDo.keys()}.")
121 for name in unblocked:
122 # Finish initialization of the element with steps that
123 # depend on those steps already having been run for all
124 # dependencies.
125 # This includes adding the element to self.elements and
126 # (if appropriate) self.dimensions.
127 elementsToDo.pop(name)._finish(self, elementsToDo)
129 # Add attributes for special subsets of the graph.
130 self.empty = DimensionGraph(self, (), conform=False)
131 self._finish()
133 # Set up factories for dataId packers as defined by config.
134 # MyPy is totally confused by us setting attributes on self in __new__.
135 self._packers = {}
136 for name, subconfig in config.get("packers", {}).items():
137 self._packers[name] = DimensionPackerFactory.fromConfig(universe=self, # type: ignore
138 config=subconfig)
140 # Use the version number from the config as a key in the singleton
141 # dict containing all instances; that will let us transfer dimension
142 # objects between processes using pickle without actually going
143 # through real initialization, as long as a universe with the same
144 # version has already been constructed in the receiving process.
145 self._version = version
146 cls._instances[self._version] = self
147 return self
149 def __repr__(self) -> str:
150 return f"DimensionUniverse({self})"
152 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph:
153 """Construct a `DimensionGraph` from a possibly-heterogenous iterable
154 of `Dimension` instances and string names thereof.
156 Constructing `DimensionGraph` directly from names or dimension
157 instances is slightly more efficient when it is known in advance that
158 the iterable is not heterogenous.
160 Parameters
161 ----------
162 iterable: iterable of `Dimension` or `str`
163 Dimensions that must be included in the returned graph (their
164 dependencies will be as well).
166 Returns
167 -------
168 graph : `DimensionGraph`
169 A `DimensionGraph` instance containing all given dimensions.
170 """
171 names = set()
172 for item in iterable:
173 try:
174 names.add(item.name) # type: ignore
175 except AttributeError:
176 names.add(item)
177 return DimensionGraph(universe=self, names=names)
179 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]:
180 """Return a sorted version of the given iterable of dimension elements.
182 The universe's sort order is topological (an element's dependencies
183 precede it), starting with skypix dimensions (which never have
184 dependencies) and then sorting lexicographically to break ties.
186 Parameters
187 ----------
188 elements : iterable of `DimensionElement`.
189 Elements to be sorted.
190 reverse : `bool`, optional
191 If `True`, sort in the opposite order.
193 Returns
194 -------
195 sorted : `list` of `DimensionElement`
196 A sorted list containing the same elements that were given.
197 """
198 s = set(elements)
199 result = [element for element in self.elements if element in s or element.name in s]
200 if reverse:
201 result.reverse()
202 # mypy thinks this can return DimensionElements even if all the user
203 # passed it was Dimensions; we know better.
204 return result # type: ignore
206 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
207 """Construct a `DimensionPacker` that can pack data ID dictionaries
208 into unique integers.
210 Parameters
211 ----------
212 name : `str`
213 Name of the packer, matching a key in the "packers" section of the
214 dimension configuration.
215 dataId : `DataCoordinate`
216 Fully-expanded data ID that identfies the at least the "fixed"
217 dimensions of the packer (i.e. those that are assumed/given,
218 setting the space over which packed integer IDs are unique).
219 ``dataId.hasRecords()`` must return `True`.
220 """
221 return self._packers[name](dataId)
223 def getEncodeLength(self) -> int:
224 """Return the size (in bytes) of the encoded size of `DimensionGraph`
225 instances in this universe.
227 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
228 information.
229 """
230 return math.ceil(len(self.dimensions)/8)
232 @classmethod
233 def _unpickle(cls, version: int) -> DimensionUniverse:
234 """Callable used for unpickling.
236 For internal use only.
237 """
238 try:
239 return cls._instances[version]
240 except KeyError as err:
241 raise pickle.UnpicklingError(
242 f"DimensionUniverse with version '{version}' "
243 f"not found. Note that DimensionUniverse objects are not "
244 f"truly serialized; when using pickle to transfer them "
245 f"between processes, an equivalent instance with the same "
246 f"version must already exist in the receiving process."
247 ) from err
249 def __reduce__(self) -> tuple:
250 return (self._unpickle, (self._version,))
252 # Class attributes below are shadowed by instance attributes, and are
253 # present just to hold the docstrings for those instance attributes.
255 empty: DimensionGraph
256 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
257 """
259 commonSkyPix: SkyPixDimension
260 """The special skypix dimension that is used to relate all other spatial
261 dimensions in the `Registry` database (`SkyPixDimension`).
262 """
264 _cache: Dict[FrozenSet[str], DimensionGraph]
266 _packers: Dict[str, DimensionPackerFactory]
268 _version: int