Coverage for python/lsst/daf/butler/dimensions/_graph.py: 63%
168 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-12 10:07 +0000
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-12 10:07 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["DimensionGraph", "SerializedDimensionGraph"]
32import warnings
33from collections.abc import Iterable, Iterator, Mapping, Set
34from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast
36import pydantic
37from deprecated.sphinx import deprecated
38from lsst.utils.classes import cached_getter, immutable
39from lsst.utils.introspection import find_outside_stacklevel
41from .._named import NamedValueAbstractSet, NameMappingSetView
42from .._topology import TopologicalFamily, TopologicalSpace
43from ..json import from_json_pydantic, to_json_pydantic
44from ._group import DimensionGroup
46if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
47 from ..registry import Registry
48 from ._elements import Dimension, DimensionElement
49 from ._governor import GovernorDimension
50 from ._skypix import SkyPixDimension
51 from ._universe import DimensionUniverse
54class SerializedDimensionGraph(pydantic.BaseModel):
55 """Simplified model of a `DimensionGraph` suitable for serialization."""
57 names: list[str]
59 @classmethod
60 def direct(cls, *, names: list[str]) -> SerializedDimensionGraph:
61 """Construct a `SerializedDimensionGraph` directly without validators.
63 Parameters
64 ----------
65 names : `list` [`str`]
66 The names of the dimensions to include.
68 Returns
69 -------
70 graph : `SerializedDimensionGraph`
71 Model representing these dimensions.
73 Notes
74 -----
75 This differs from the pydantic "construct" method in that the arguments
76 are explicitly what the model requires, and it will recurse through
77 members, constructing them from their corresponding `direct` methods.
79 This method should only be called when the inputs are trusted.
80 """
81 return cls.model_construct(names=names)
84_T = TypeVar("_T", bound="DimensionElement", covariant=True)
87# TODO: Remove on DM-41326.
88_NVAS_DEPRECATION_MSG = """DimensionGraph is deprecated in favor of
89DimensionGroup, which uses sets of str names instead of NamedValueAbstractSets
90of Dimension or DimensionElement instances. Support for the
91NamedValueAbstractSet interfaces on this object will be dropped after v27.
92"""
95class _DimensionGraphNamedValueSet(NameMappingSetView[_T]):
96 def __init__(self, keys: Set[str], universe: DimensionUniverse):
97 super().__init__({k: cast(_T, universe[k]) for k in keys})
99 # TODO: Remove on DM-41326.
100 @deprecated(
101 _NVAS_DEPRECATION_MSG
102 + "Use a dict comprehension and DimensionUniverse indexing to construct a mapping when needed.",
103 version="v27",
104 category=FutureWarning,
105 )
106 def asMapping(self) -> Mapping[str, _T]:
107 return super().asMapping()
109 # TODO: Remove on DM-41326.
110 @deprecated(
111 _NVAS_DEPRECATION_MSG + "Use DimensionUniverse for DimensionElement lookups.",
112 version="v27",
113 category=FutureWarning,
114 )
115 def __getitem__(self, key: str | _T) -> _T:
116 return super().__getitem__(key)
118 def __contains__(self, key: Any) -> bool:
119 from ._elements import DimensionElement
121 if isinstance(key, DimensionElement):
122 warnings.warn(
123 _NVAS_DEPRECATION_MSG + "'in' expressions must use str keys.",
124 category=FutureWarning,
125 stacklevel=find_outside_stacklevel("lsst.daf.butler."),
126 )
127 return super().__contains__(key)
129 def __iter__(self) -> Iterator[_T]:
130 # TODO: Remove on DM-41326.
131 warnings.warn(
132 _NVAS_DEPRECATION_MSG
133 + (
134 "In the future, iteration will yield str names; for now, use .names "
135 "to do the same without triggering this warning."
136 ),
137 category=FutureWarning,
138 stacklevel=find_outside_stacklevel("lsst.daf.butler."),
139 )
140 return super().__iter__()
142 def __eq__(self, other: Any) -> bool:
143 # TODO: Remove on DM-41326.
144 warnings.warn(
145 _NVAS_DEPRECATION_MSG
146 + (
147 "In the future, set-equality will assume str keys; for now, use .names "
148 "to do the same without triggering this warning."
149 ),
150 category=FutureWarning,
151 stacklevel=find_outside_stacklevel("lsst.daf.butler."),
152 )
153 return super().__eq__(other)
155 def __le__(self, other: Set[Any]) -> bool:
156 # TODO: Remove on DM-41326.
157 warnings.warn(
158 _NVAS_DEPRECATION_MSG
159 + (
160 "In the future, subset tests will assume str keys; for now, use .names "
161 "to do the same without triggering this warning."
162 ),
163 category=FutureWarning,
164 stacklevel=find_outside_stacklevel("lsst.daf.butler."),
165 )
166 return super().__le__(other)
168 def __ge__(self, other: Set[Any]) -> bool:
169 # TODO: Remove on DM-41326.
170 warnings.warn(
171 _NVAS_DEPRECATION_MSG
172 + (
173 "In the future, superset tests will assume str keys; for now, use .names "
174 "to do the same without triggering this warning."
175 ),
176 category=FutureWarning,
177 stacklevel=find_outside_stacklevel("lsst.daf.butler."),
178 )
179 return super().__ge__(other)
182# TODO: Remove on DM-41326.
183@deprecated(
184 "DimensionGraph is deprecated in favor of DimensionGroup and will be removed after v27.",
185 category=FutureWarning,
186 version="v27",
187)
188@immutable
189class DimensionGraph: # numpydoc ignore=PR02
190 """An immutable, dependency-complete collection of dimensions.
192 `DimensionGraph` is deprecated in favor of `DimensionGroup` and will be
193 removed after v27. The two types have very similar interfaces, but
194 `DimensionGroup` does not support direct iteration and its set-like
195 attributes are of dimension element names, not `DimensionElement`
196 instances. `DimensionGraph` objects are still returned by certain
197 non-deprecated methods and properties (most prominently
198 `DatasetType.dimensions`), and to handle these cases deprecation warnings
199 are only emitted for operations on `DimensionGraph` that are not
200 supported by `DimensionGroup` as well.
202 Parameters
203 ----------
204 universe : `DimensionUniverse`
205 The special graph of all known dimensions of which this graph will be a
206 subset.
207 dimensions : iterable of `Dimension`, optional
208 An iterable of `Dimension` instances that must be included in the
209 graph. All (recursive) dependencies of these dimensions will also be
210 included. At most one of ``dimensions`` and ``names`` must be
211 provided.
212 names : iterable of `str`, optional
213 An iterable of the names of dimensions that must be included in the
214 graph. All (recursive) dependencies of these dimensions will also be
215 included. At most one of ``dimensions`` and ``names`` must be
216 provided.
217 conform : `bool`, optional
218 If `True` (default), expand to include dependencies. `False` should
219 only be used for callers that can guarantee that other arguments are
220 already correctly expanded, and is primarily for internal use.
222 Notes
223 -----
224 `DimensionGraph` should be used instead of other collections in most
225 contexts where a collection of dimensions is required and a
226 `DimensionUniverse` is available. Exceptions include cases where order
227 matters (and is different from the consistent ordering defined by the
228 `DimensionUniverse`), or complete `~collection.abc.Set` semantics are
229 required.
230 """
232 _serializedType = SerializedDimensionGraph
234 def __new__(
235 cls,
236 universe: DimensionUniverse,
237 dimensions: Iterable[Dimension] | None = None,
238 names: Iterable[str] | None = None,
239 conform: bool = True,
240 ) -> DimensionGraph:
241 if names is None:
242 if dimensions is None:
243 group = DimensionGroup(universe)
244 else:
245 group = DimensionGroup(universe, {d.name for d in dimensions}, _conform=conform)
246 else:
247 if dimensions is not None:
248 raise TypeError("Only one of 'dimensions' and 'names' may be provided.")
249 group = DimensionGroup(universe, names, _conform=conform)
250 return group._as_graph()
252 @property
253 def universe(self) -> DimensionUniverse:
254 """Object that manages all known dimensions."""
255 return self._group.universe
257 @property
258 @deprecated(
259 _NVAS_DEPRECATION_MSG + "Use '.names' instead of '.dimensions' or '.dimensions.names'.",
260 version="v27",
261 category=FutureWarning,
262 )
263 @cached_getter
264 def dimensions(self) -> NamedValueAbstractSet[Dimension]:
265 """A true `~collections.abc.Set` of all true `Dimension` instances in
266 the graph.
267 """
268 return _DimensionGraphNamedValueSet(self._group.names, self._group.universe)
270 @property
271 @cached_getter
272 def elements(self) -> NamedValueAbstractSet[DimensionElement]:
273 """A true `~collections.abc.Set` of all `DimensionElement` instances in
274 the graph; a superset of `dimensions` (`NamedValueAbstractSet` of
275 `DimensionElement`).
276 """
277 return _DimensionGraphNamedValueSet(self._group.elements, self._group.universe)
279 @property
280 @cached_getter
281 def governors(self) -> NamedValueAbstractSet[GovernorDimension]:
282 """A true `~collections.abc.Set` of all `GovernorDimension` instances
283 in the graph.
284 """
285 return _DimensionGraphNamedValueSet(self._group.governors, self._group.universe)
287 @property
288 @cached_getter
289 def skypix(self) -> NamedValueAbstractSet[SkyPixDimension]:
290 """A true `~collections.abc.Set` of all `SkyPixDimension` instances
291 in the graph.
292 """
293 return _DimensionGraphNamedValueSet(self._group.skypix, self._group.universe)
295 @property
296 @cached_getter
297 def required(self) -> NamedValueAbstractSet[Dimension]:
298 """The subset of `dimensions` whose elements must be directly
299 identified via their primary keys in a data ID in order to identify the
300 rest of the elements in the graph.
301 """
302 return _DimensionGraphNamedValueSet(self._group.required, self._group.universe)
304 @property
305 @cached_getter
306 def implied(self) -> NamedValueAbstractSet[Dimension]:
307 """The subset of `dimensions` whose elements need not be directly
308 identified via their primary keys in a data ID.
309 """
310 return _DimensionGraphNamedValueSet(self._group.implied, self._group.universe)
312 def __getnewargs__(self) -> tuple:
313 return (self.universe, None, tuple(self._group.names), False)
315 def __deepcopy__(self, memo: dict) -> DimensionGraph:
316 # DimensionGraph is recursively immutable; see note in @immutable
317 # decorator.
318 return self
320 @property
321 def names(self) -> Set[str]:
322 """Set of the names of all dimensions in the graph."""
323 return self._group.names
325 def to_simple(self, minimal: bool = False) -> SerializedDimensionGraph:
326 """Convert this class to a simple python type.
328 This type is suitable for serialization.
330 Parameters
331 ----------
332 minimal : `bool`, optional
333 Use minimal serialization. Has no effect on for this class.
335 Returns
336 -------
337 names : `list`
338 The names of the dimensions.
339 """
340 # Names are all we can serialize.
341 return SerializedDimensionGraph(names=list(self.names))
343 @classmethod
344 def from_simple(
345 cls,
346 names: SerializedDimensionGraph,
347 universe: DimensionUniverse | None = None,
348 registry: Registry | None = None,
349 ) -> DimensionGraph:
350 """Construct a new object from the simplified form.
352 This is assumed to support data data returned from the `to_simple`
353 method.
355 Parameters
356 ----------
357 names : `list` of `str`
358 The names of the dimensions.
359 universe : `DimensionUniverse`
360 The special graph of all known dimensions of which this graph will
361 be a subset. Can be `None` if `Registry` is provided.
362 registry : `lsst.daf.butler.Registry`, optional
363 Registry from which a universe can be extracted. Can be `None`
364 if universe is provided explicitly.
366 Returns
367 -------
368 graph : `DimensionGraph`
369 Newly-constructed object.
370 """
371 if universe is None and registry is None:
372 raise ValueError("One of universe or registry is required to convert names to a DimensionGraph")
373 if universe is None and registry is not None:
374 universe = registry.dimensions
375 if universe is None:
376 # this is for mypy
377 raise ValueError("Unable to determine a usable universe")
379 return cls(names=names.names, universe=universe)
381 to_json = to_json_pydantic
382 from_json: ClassVar = classmethod(from_json_pydantic)
384 def __iter__(self) -> Iterator[Dimension]:
385 """Iterate over all dimensions in the graph.
387 (and true `Dimension` instances only).
388 """
389 return iter(self.dimensions)
391 def __len__(self) -> int:
392 """Return the number of dimensions in the graph.
394 (and true `Dimension` instances only).
395 """
396 return len(self._group)
398 def __contains__(self, element: str | DimensionElement) -> bool:
399 """Return `True` if the given element or element name is in the graph.
401 This test covers all `DimensionElement` instances in ``self.elements``,
402 not just true `Dimension` instances).
403 """
404 return element in self.elements
406 def __getitem__(self, name: str) -> DimensionElement:
407 """Return the element with the given name.
409 This lookup covers all `DimensionElement` instances in
410 ``self.elements``, not just true `Dimension` instances).
411 """
412 return self.elements[name]
414 def get(self, name: str, default: Any = None) -> DimensionElement:
415 """Return the element with the given name.
417 This lookup covers all `DimensionElement` instances in
418 ``self.elements``, not just true `Dimension` instances).
420 Parameters
421 ----------
422 name : `str`
423 Name of element to return.
424 default : `typing.Any` or `None`
425 Default value if named element is not present.
427 Returns
428 -------
429 element : `DimensionElement` or `None`
430 The element found, or the default.
431 """
432 return self.elements.get(name, default)
434 def __str__(self) -> str:
435 return str(self.as_group())
437 def __repr__(self) -> str:
438 return f"DimensionGraph({str(self)})"
440 def as_group(self) -> DimensionGroup:
441 """Return a `DimensionGroup` that represents the same set of
442 dimensions.
444 Returns
445 -------
446 group : `DimensionGroup`
447 Group that represents the same set of dimensions.
448 """
449 return self._group
451 def isdisjoint(self, other: DimensionGroup | DimensionGraph) -> bool:
452 """Test whether the intersection of two graphs is empty.
454 Parameters
455 ----------
456 other : `DimensionGroup` or `DimensionGraph`
457 Other graph to compare with.
459 Returns
460 -------
461 is_disjoint : `bool`
462 Returns `True` if either operand is the empty.
463 """
464 return self._group.isdisjoint(other.as_group())
466 def issubset(self, other: DimensionGroup | DimensionGraph) -> bool:
467 """Test whether all dimensions in ``self`` are also in ``other``.
469 Parameters
470 ----------
471 other : `DimensionGroup` or `DimensionGraph`
472 Other graph to compare with.
474 Returns
475 -------
476 is_subset : `bool`
477 Returns `True` if ``self`` is empty.
478 """
479 return self._group <= other.as_group()
481 def issuperset(self, other: DimensionGroup | DimensionGraph) -> bool:
482 """Test whether all dimensions in ``other`` are also in ``self``.
484 Parameters
485 ----------
486 other : `DimensionGroup` or `DimensionGraph`
487 Other graph to compare with.
489 Returns
490 -------
491 is_superset : `bool`
492 Returns `True` if ``other`` is empty.
493 """
494 return self._group >= other.as_group()
496 def __eq__(self, other: Any) -> bool:
497 """Test the arguments have exactly the same dimensions & elements."""
498 if isinstance(other, (DimensionGraph, DimensionGroup)):
499 return self._group == other.as_group()
500 return False
502 def __hash__(self) -> int:
503 return hash(self.as_group())
505 def __le__(self, other: DimensionGroup | DimensionGraph) -> bool:
506 """Test whether ``self`` is a subset of ``other``."""
507 return self._group <= other.as_group()
509 def __ge__(self, other: DimensionGroup | DimensionGraph) -> bool:
510 """Test whether ``self`` is a superset of ``other``."""
511 return self._group >= other.as_group()
513 def __lt__(self, other: DimensionGroup | DimensionGraph) -> bool:
514 """Test whether ``self`` is a strict subset of ``other``."""
515 return self._group < other.as_group()
517 def __gt__(self, other: DimensionGroup | DimensionGraph) -> bool:
518 """Test whether ``self`` is a strict superset of ``other``."""
519 return self._group > other.as_group()
521 def union(self, *others: DimensionGroup | DimensionGraph) -> DimensionGraph:
522 """Construct a new graph with all dimensions in any of the operands.
524 Parameters
525 ----------
526 *others : `DimensionGroup` or `DimensionGraph`
527 Other graphs to join with.
529 Returns
530 -------
531 union : `DimensionGraph`
532 The union of this graph wit hall the others.
534 Notes
535 -----
536 The elements of the returned graph may exceed the naive union of
537 their elements, as some `DimensionElement` instances are included
538 in graphs whenever multiple dimensions are present, and those
539 dependency dimensions could have been provided by different operands.
540 """
541 names = set(self.names).union(*[other.names for other in others])
542 return self.universe.conform(names)._as_graph()
544 def intersection(self, *others: DimensionGroup | DimensionGraph) -> DimensionGraph:
545 """Construct a new graph with only dimensions in all of the operands.
547 Parameters
548 ----------
549 *others : `DimensionGroup` or `DimensionGraph`
550 Other graphs to use.
552 Returns
553 -------
554 inter : `DimensionGraph`
555 Intersection of all the graphs.
557 Notes
558 -----
559 See also `union`.
560 """
561 names = set(self.names).intersection(*[other.names for other in others])
562 return self.universe.conform(names)._as_graph()
564 def __or__(self, other: DimensionGroup | DimensionGraph) -> DimensionGraph:
565 """Construct a new graph with all dimensions in any of the operands.
567 See `union`.
568 """
569 return self.union(other)
571 def __and__(self, other: DimensionGroup | DimensionGraph) -> DimensionGraph:
572 """Construct a new graph with only dimensions in all of the operands.
574 See `intersection`.
575 """
576 return self.intersection(other)
578 # TODO: Remove on DM-41326.
579 @property
580 @deprecated(
581 "DimensionGraph is deprecated in favor of DimensionGroup, which does not have this attribute; "
582 "use .lookup_order. DimensionGraph will be removed after v27.",
583 category=FutureWarning,
584 version="v27",
585 )
586 def primaryKeyTraversalOrder(self) -> tuple[DimensionElement, ...]:
587 """A tuple of all elements in specific order.
589 The order allows records to be found given their primary keys, starting
590 from only the primary keys of required dimensions (`tuple` [
591 `DimensionRecord` ]).
593 Unlike the table definition/topological order (which is what
594 DimensionUniverse.sorted gives you), when dimension A implies dimension
595 B, dimension A appears first.
596 """
597 return tuple(self.universe[element_name] for element_name in self._group.lookup_order)
599 @property
600 def spatial(self) -> NamedValueAbstractSet[TopologicalFamily]:
601 """Families represented by the spatial elements in this graph."""
602 return self._group.spatial
604 @property
605 def temporal(self) -> NamedValueAbstractSet[TopologicalFamily]:
606 """Families represented by the temporal elements in this graph."""
607 return self._group.temporal
609 # TODO: Remove on DM-41326.
610 @property
611 @deprecated(
612 "DimensionGraph is deprecated in favor of DimensionGroup, which does not have this attribute; "
613 "use .spatial or .temporal. DimensionGraph will be removed after v27.",
614 category=FutureWarning,
615 version="v27",
616 )
617 def topology(self) -> Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]:
618 """Families of elements in this graph that can participate in
619 topological relationships.
620 """
621 return self._group._space_families
623 _group: DimensionGroup