Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 40%
165 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["DimensionUniverse"]
32import logging
33import math
34import pickle
35from collections import defaultdict
36from collections.abc import Iterable, Mapping
37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar
39from deprecated.sphinx import deprecated
40from lsst.utils.classes import cached_getter, immutable
42from .._topology import TopologicalFamily, TopologicalSpace
43from ..config import Config
44from ..named import NamedValueAbstractSet, NamedValueSet
45from ._config import _DEFAULT_NAMESPACE, DimensionConfig
46from ._database import DatabaseDimensionElement
47from ._elements import Dimension, DimensionElement
48from ._governor import GovernorDimension
49from ._graph import DimensionGraph
50from ._skypix import SkyPixDimension, SkyPixSystem
52if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
53 from ._coordinate import DataCoordinate
54 from ._packer import DimensionPacker, DimensionPackerFactory
55 from .construction import DimensionConstructionBuilder
58E = TypeVar("E", bound=DimensionElement)
59_LOG = logging.getLogger(__name__)
62@immutable
63class DimensionUniverse:
64 """Self-consistent set of dimensions.
66 A parent class that represents a complete, self-consistent set of
67 dimensions and their relationships.
69 `DimensionUniverse` is not a class-level singleton, but all instances are
70 tracked in a singleton map keyed by the version number and namespace
71 in the configuration they were loaded from. Because these universes
72 are solely responsible for constructing `DimensionElement` instances,
73 these are also indirectly tracked by that singleton as well.
75 Parameters
76 ----------
77 config : `Config`, optional
78 Configuration object from which dimension definitions can be extracted.
79 Ignored if ``builder`` is provided, or if ``version`` is provided and
80 an instance with that version already exists.
81 version : `int`, optional
82 Integer version for this `DimensionUniverse`. If not provided, a
83 version will be obtained from ``builder`` or ``config``.
84 namespace : `str`, optional
85 Namespace of this `DimensionUniverse`, combined with the version
86 to provide universe safety for registries that use different
87 dimension definitions.
88 builder : `DimensionConstructionBuilder`, optional
89 Builder object used to initialize a new instance. Ignored if
90 ``version`` is provided and an instance with that version already
91 exists. Should not have had `~DimensionConstructionBuilder.finish`
92 called; this will be called if needed by `DimensionUniverse`.
93 """
95 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {}
96 """Singleton dictionary of all instances, keyed by version.
98 For internal use only.
99 """
101 def __new__(
102 cls,
103 config: Config | None = None,
104 *,
105 version: int | None = None,
106 namespace: str | None = None,
107 builder: DimensionConstructionBuilder | None = None,
108 ) -> DimensionUniverse:
109 # Try to get a version first, to look for existing instances; try to
110 # do as little work as possible at this stage.
111 if version is None:
112 if builder is None:
113 config = DimensionConfig(config)
114 version = config["version"]
115 else:
116 version = builder.version
118 # Then a namespace.
119 if namespace is None:
120 if builder is None:
121 config = DimensionConfig(config)
122 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
123 else:
124 namespace = builder.namespace
125 # if still None use the default
126 if namespace is None:
127 namespace = _DEFAULT_NAMESPACE
129 # See if an equivalent instance already exists.
130 self: DimensionUniverse | None = cls._instances.get((version, namespace))
131 if self is not None:
132 return self
134 # Ensure we have a builder, building one from config if necessary.
135 if builder is None:
136 config = DimensionConfig(config)
137 builder = config.makeBuilder()
139 # Delegate to the builder for most of the construction work.
140 builder.finish()
142 # Create the universe instance and create core attributes, mostly
143 # copying from builder.
144 self = object.__new__(cls)
145 assert self is not None
146 self._cache = {}
147 self._dimensions = builder.dimensions
148 self._elements = builder.elements
149 self._topology = builder.topology
150 self._packers = builder.packers
151 self.dimensionConfig = builder.config
152 commonSkyPix = self._dimensions[builder.commonSkyPixName]
153 assert isinstance(commonSkyPix, SkyPixDimension)
154 self.commonSkyPix = commonSkyPix
156 # Attach self to all elements.
157 for element in self._elements:
158 element.universe = self
160 # Add attribute for special subsets of the graph.
161 self.empty = DimensionGraph(self, (), conform=False)
163 # Use the version number and namespace from the config as a key in
164 # the singleton dict containing all instances; that will let us
165 # transfer dimension objects between processes using pickle without
166 # actually going through real initialization, as long as a universe
167 # with the same version and namespace has already been constructed in
168 # the receiving process.
169 self._version = version
170 self._namespace = namespace
171 cls._instances[self._version, self._namespace] = self
173 # Build mappings from element to index. These are used for
174 # topological-sort comparison operators in DimensionElement itself.
175 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
176 # Same for dimension to index, sorted topologically across required
177 # and implied. This is used for encode/decode.
178 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
180 self._populates = defaultdict(NamedValueSet)
181 for element in self._elements:
182 if element.populated_by is not None:
183 self._populates[element.populated_by.name].add(element)
185 return self
187 @property
188 def version(self) -> int:
189 """The version number of this universe.
191 Returns
192 -------
193 version : `int`
194 An integer representing the version number of this universe.
195 Uniquely defined when combined with the `namespace`.
196 """
197 return self._version
199 @property
200 def namespace(self) -> str:
201 """The namespace associated with this universe.
203 Returns
204 -------
205 namespace : `str`
206 The namespace. When combined with the `version` can uniquely
207 define this universe.
208 """
209 return self._namespace
211 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
212 """Check compatibility between this `DimensionUniverse` and another.
214 Parameters
215 ----------
216 other : `DimensionUniverse`
217 The other `DimensionUniverse` to check for compatibility
219 Returns
220 -------
221 results : `bool`
222 If the other `DimensionUniverse` is compatible with this one return
223 `True`, else `False`
224 """
225 # Different namespaces mean that these universes cannot be compatible.
226 if self.namespace != other.namespace:
227 return False
228 if self.version != other.version:
229 _LOG.info(
230 "Universes share a namespace %r but have differing versions (%d != %d). "
231 " This could be okay but may be responsible for dimension errors later.",
232 self.namespace,
233 self.version,
234 other.version,
235 )
237 # For now assume compatibility if versions differ.
238 return True
240 def __repr__(self) -> str:
241 return f"DimensionUniverse({self._version}, {self._namespace})"
243 def __getitem__(self, name: str) -> DimensionElement:
244 return self._elements[name]
246 def __contains__(self, name: Any) -> bool:
247 return name in self._elements
249 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None:
250 """Return the `DimensionElement` with the given name or a default.
252 Parameters
253 ----------
254 name : `str`
255 Name of the element.
256 default : `DimensionElement`, optional
257 Element to return if the named one does not exist. Defaults to
258 `None`.
260 Returns
261 -------
262 element : `DimensionElement`
263 The named element.
264 """
265 return self._elements.get(name, default)
267 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
268 """Return a set of all static elements in this universe.
270 Non-static elements that are created as needed may also exist, but
271 these are guaranteed to have no direct relationships to other elements
272 (though they may have spatial or temporal relationships).
274 Returns
275 -------
276 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
277 A frozen set of `DimensionElement` instances.
278 """
279 return self._elements
281 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
282 """Return a set of all static dimensions in this universe.
284 Non-static dimensions that are created as needed may also exist, but
285 these are guaranteed to have no direct relationships to other elements
286 (though they may have spatial or temporal relationships).
288 Returns
289 -------
290 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
291 A frozen set of `Dimension` instances.
292 """
293 return self._dimensions
295 @cached_getter
296 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
297 """Return a set of all `GovernorDimension` instances in this universe.
299 Returns
300 -------
301 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
302 A frozen set of `GovernorDimension` instances.
303 """
304 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
306 @cached_getter
307 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
308 """Return set of all `DatabaseDimensionElement` instances in universe.
310 This does not include `GovernorDimension` instances, which are backed
311 by the database but do not inherit from `DatabaseDimensionElement`.
313 Returns
314 -------
315 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
316 A frozen set of `DatabaseDimensionElement` instances.
317 """
318 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
320 @property
321 @cached_getter
322 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
323 """All skypix systems known to this universe.
325 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
326 """
327 return NamedValueSet(
328 [
329 family
330 for family in self._topology[TopologicalSpace.SPATIAL]
331 if isinstance(family, SkyPixSystem)
332 ]
333 ).freeze()
335 def getElementIndex(self, name: str) -> int:
336 """Return the position of the named dimension element.
338 The position is in this universe's sorting of all elements.
340 Parameters
341 ----------
342 name : `str`
343 Name of the element.
345 Returns
346 -------
347 index : `int`
348 Sorting index for this element.
349 """
350 return self._elementIndices[name]
352 def getDimensionIndex(self, name: str) -> int:
353 """Return the position of the named dimension.
355 This position is in this universe's sorting of all dimensions.
357 Parameters
358 ----------
359 name : `str`
360 Name of the dimension.
362 Returns
363 -------
364 index : `int`
365 Sorting index for this dimension.
367 Notes
368 -----
369 The dimension sort order for a universe is consistent with the element
370 order (all dimensions are elements), and either can be used to sort
371 dimensions if used consistently. But there are also some contexts in
372 which contiguous dimension-only indices are necessary or at least
373 desirable.
374 """
375 return self._dimensionIndices[name]
377 def expandDimensionNameSet(self, names: set[str]) -> None:
378 """Expand a set of dimension names in-place.
380 Includes recursive dependencies.
382 This is an advanced interface for cases where constructing a
383 `DimensionGraph` (which also expands required dependencies) is
384 impossible or undesirable.
386 Parameters
387 ----------
388 names : `set` [ `str` ]
389 A true `set` of dimension names, to be expanded in-place.
390 """
391 # Keep iterating until the set of names stops growing. This is not as
392 # efficient as it could be, but we work pretty hard cache
393 # DimensionGraph instances to keep actual construction rare, so that
394 # shouldn't matter.
395 oldSize = len(names)
396 while True:
397 # iterate over a temporary copy so we can modify the original
398 for name in tuple(names):
399 names.update(self._dimensions[name].required.names)
400 names.update(self._dimensions[name].implied.names)
401 if oldSize == len(names):
402 break
403 else:
404 oldSize = len(names)
406 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph:
407 """Construct graph from iterable.
409 Constructs a `DimensionGraph` from a possibly-heterogenous iterable
410 of `Dimension` instances and string names thereof.
412 Constructing `DimensionGraph` directly from names or dimension
413 instances is slightly more efficient when it is known in advance that
414 the iterable is not heterogenous.
416 Parameters
417 ----------
418 iterable: iterable of `Dimension` or `str`
419 Dimensions that must be included in the returned graph (their
420 dependencies will be as well).
422 Returns
423 -------
424 graph : `DimensionGraph`
425 A `DimensionGraph` instance containing all given dimensions.
426 """
427 names = set()
428 for item in iterable:
429 try:
430 names.add(item.name) # type: ignore
431 except AttributeError:
432 names.add(item)
433 return DimensionGraph(universe=self, names=names)
435 def sorted(self, elements: Iterable[E | str], *, reverse: bool = False) -> list[E]:
436 """Return a sorted version of the given iterable of dimension elements.
438 The universe's sort order is topological (an element's dependencies
439 precede it), with an unspecified (but deterministic) approach to
440 breaking ties.
442 Parameters
443 ----------
444 elements : iterable of `DimensionElement`.
445 Elements to be sorted.
446 reverse : `bool`, optional
447 If `True`, sort in the opposite order.
449 Returns
450 -------
451 sorted : `list` of `DimensionElement`
452 A sorted list containing the same elements that were given.
453 """
454 s = set(elements)
455 result = [element for element in self._elements if element in s or element.name in s]
456 if reverse:
457 result.reverse()
458 # mypy thinks this can return DimensionElements even if all the user
459 # passed it was Dimensions; we know better.
460 return result # type: ignore
462 # TODO: Remove this method on DM-38687.
463 @deprecated(
464 "Deprecated in favor of configurable dimension packers. Will be removed after v26.",
465 version="v26",
466 category=FutureWarning,
467 )
468 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker:
469 """Make a dimension packer.
471 Constructs a `DimensionPacker` that can pack data ID dictionaries
472 into unique integers.
474 Parameters
475 ----------
476 name : `str`
477 Name of the packer, matching a key in the "packers" section of the
478 dimension configuration.
479 dataId : `DataCoordinate`
480 Fully-expanded data ID that identifies the at least the "fixed"
481 dimensions of the packer (i.e. those that are assumed/given,
482 setting the space over which packed integer IDs are unique).
483 ``dataId.hasRecords()`` must return `True`.
484 """
485 return self._packers[name](self, dataId)
487 def getEncodeLength(self) -> int:
488 """Return encoded size of graph.
490 Returns the size (in bytes) of the encoded size of `DimensionGraph`
491 instances in this universe.
493 See `DimensionGraph.encode` and `DimensionGraph.decode` for more
494 information.
495 """
496 return math.ceil(len(self._dimensions) / 8)
498 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]:
499 """Return the set of `DimensionElement` objects whose
500 `~DimensionElement.populated_by` atttribute is the given dimension.
501 """
502 return self._populates[dimension.name]
504 @classmethod
505 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse:
506 """Return an unpickled dimension universe.
508 Callable used for unpickling.
510 For internal use only.
511 """
512 if namespace is None:
513 # Old pickled universe.
514 namespace = _DEFAULT_NAMESPACE
515 try:
516 return cls._instances[version, namespace]
517 except KeyError as err:
518 raise pickle.UnpicklingError(
519 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
520 "not found. Note that DimensionUniverse objects are not "
521 "truly serialized; when using pickle to transfer them "
522 "between processes, an equivalent instance with the same "
523 "version must already exist in the receiving process."
524 ) from err
526 def __reduce__(self) -> tuple:
527 return (self._unpickle, (self._version, self._namespace))
529 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
530 # DimensionUniverse is recursively immutable; see note in @immutable
531 # decorator.
532 return self
534 # Class attributes below are shadowed by instance attributes, and are
535 # present just to hold the docstrings for those instance attributes.
537 empty: DimensionGraph
538 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`).
539 """
541 commonSkyPix: SkyPixDimension
542 """The special skypix dimension that is used to relate all other spatial
543 dimensions in the `Registry` database (`SkyPixDimension`).
544 """
546 dimensionConfig: DimensionConfig
547 """The configuration used to create this Universe (`DimensionConfig`)."""
549 _cache: dict[frozenset[str], DimensionGraph]
551 _dimensions: NamedValueAbstractSet[Dimension]
553 _elements: NamedValueAbstractSet[DimensionElement]
555 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
557 _dimensionIndices: dict[str, int]
559 _elementIndices: dict[str, int]
561 _packers: dict[str, DimensionPackerFactory]
563 _populates: defaultdict[str, NamedValueSet[DimensionElement]]
565 _version: int
567 _namespace: str