Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 33%
355 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-12 09:20 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-12 09:20 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
23# Design notes for this module are in
24# doc/lsst.daf.butler/dev/dataCoordinate.py.
25#
27from __future__ import annotations
29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate")
31import numbers
32from abc import abstractmethod
33from collections.abc import Iterator, Mapping, Set
34from typing import TYPE_CHECKING, Any, ClassVar, Literal, overload
36from deprecated.sphinx import deprecated
37from lsst.daf.butler._compat import _BaseModelCompat
38from lsst.sphgeom import IntersectionRegion, Region
40from ..json import from_json_pydantic, to_json_pydantic
41from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping
42from ..persistenceContext import PersistenceContextVars
43from ..timespan import Timespan
44from ._elements import Dimension, DimensionElement
45from ._graph import DimensionGraph
46from ._records import DimensionRecord, SerializedDimensionRecord
48if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
49 from ...registry import Registry
50 from ._universe import DimensionUniverse
52DataIdKey = str | Dimension
53"""Type annotation alias for the keys that can be used to index a
54DataCoordinate.
55"""
57# Pydantic will cast int to str if str is first in the Union.
58DataIdValue = int | str | None
59"""Type annotation alias for the values that can be present in a
60DataCoordinate or other data ID.
61"""
64class SerializedDataCoordinate(_BaseModelCompat):
65 """Simplified model for serializing a `DataCoordinate`."""
67 dataId: dict[str, DataIdValue]
68 records: dict[str, SerializedDimensionRecord] | None = None
70 @classmethod
71 def direct(
72 cls, *, dataId: dict[str, DataIdValue], records: dict[str, dict] | None
73 ) -> SerializedDataCoordinate:
74 """Construct a `SerializedDataCoordinate` directly without validators.
76 This differs from the pydantic "construct" method in that the arguments
77 are explicitly what the model requires, and it will recurse through
78 members, constructing them from their corresponding `direct` methods.
80 This method should only be called when the inputs are trusted.
81 """
82 key = (frozenset(dataId.items()), records is not None)
83 cache = PersistenceContextVars.serializedDataCoordinateMapping.get()
84 if cache is not None and (result := cache.get(key)) is not None:
85 return result
87 if records is None:
88 serialized_records = None
89 else:
90 serialized_records = {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()}
92 node = cls.model_construct(dataId=dataId, records=serialized_records)
94 if cache is not None:
95 cache[key] = node
96 return node
99def _intersectRegions(*args: Region) -> Region | None:
100 """Return the intersection of several regions.
102 For internal use by `ExpandedDataCoordinate` only.
104 If no regions are provided, returns `None`.
105 """
106 if len(args) == 0:
107 return None
108 else:
109 result = args[0]
110 for n in range(1, len(args)):
111 result = IntersectionRegion(result, args[n])
112 return result
115class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]):
116 """Data ID dictionary.
118 An immutable data ID dictionary that guarantees that its key-value pairs
119 identify at least all required dimensions in a `DimensionGraph`.
121 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory
122 functions for private concrete implementations that should be sufficient
123 for most purposes. `standardize` is the most flexible and safe of these;
124 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are
125 more specialized and perform little or no checking of inputs.
127 Notes
128 -----
129 Like any data ID class, `DataCoordinate` behaves like a dictionary, but
130 with some subtleties:
132 - Both `Dimension` instances and `str` names thereof may be used as keys
133 in lookup operations, but iteration (and `keys`) will yield `Dimension`
134 instances. The `names` property can be used to obtain the corresponding
135 `str` names.
137 - Lookups for implied dimensions (those in ``self.graph.implied``) are
138 supported if and only if `hasFull` returns `True`, and are never
139 included in iteration or `keys`. The `full` property may be used to
140 obtain a mapping whose keys do include implied dimensions.
142 - Equality comparison with other mappings is supported, but it always
143 considers only required dimensions (as well as requiring both operands
144 to identify the same dimensions). This is not quite consistent with the
145 way mappings usually work - normally differing keys imply unequal
146 mappings - but it makes sense in this context because data IDs with the
147 same values for required dimensions but different values for implied
148 dimensions represent a serious problem with the data that
149 `DataCoordinate` cannot generally recognize on its own, and a data ID
150 that knows implied dimension values should still be able to compare as
151 equal to one that does not. This is of course not the way comparisons
152 between simple `dict` data IDs work, and hence using a `DataCoordinate`
153 instance for at least one operand in any data ID comparison is strongly
154 recommended.
156 See Also
157 --------
158 :ref:`lsst.daf.butler-dimensions_data_ids`
159 """
161 __slots__ = ()
163 _serializedType = SerializedDataCoordinate
165 @staticmethod
166 def standardize(
167 mapping: NameLookupMapping[Dimension, DataIdValue] | None = None,
168 *,
169 graph: DimensionGraph | None = None,
170 universe: DimensionUniverse | None = None,
171 defaults: DataCoordinate | None = None,
172 **kwargs: Any,
173 ) -> DataCoordinate:
174 """Standardize the supplied dataId.
176 Adapts an arbitrary mapping and/or additional arguments into a true
177 `DataCoordinate`, or augment an existing one.
179 Parameters
180 ----------
181 mapping : `~collections.abc.Mapping`, optional
182 An informal data ID that maps dimensions or dimension names to
183 their primary key values (may also be a true `DataCoordinate`).
184 graph : `DimensionGraph`
185 The dimensions to be identified by the new `DataCoordinate`.
186 If not provided, will be inferred from the keys of ``mapping`` and
187 ``**kwargs``, and ``universe`` must be provided unless ``mapping``
188 is already a `DataCoordinate`.
189 universe : `DimensionUniverse`
190 All known dimensions and their relationships; used to expand
191 and validate dependencies when ``graph`` is not provided.
192 defaults : `DataCoordinate`, optional
193 Default dimension key-value pairs to use when needed. These are
194 never used to infer ``graph``, and are ignored if a different value
195 is provided for the same key in ``mapping`` or `**kwargs``.
196 **kwargs
197 Additional keyword arguments are treated like additional key-value
198 pairs in ``mapping``.
200 Returns
201 -------
202 coordinate : `DataCoordinate`
203 A validated `DataCoordinate` instance.
205 Raises
206 ------
207 TypeError
208 Raised if the set of optional arguments provided is not supported.
209 KeyError
210 Raised if a key-value pair for a required dimension is missing.
211 """
212 d: dict[str, DataIdValue] = {}
213 if isinstance(mapping, DataCoordinate):
214 if graph is None:
215 if not kwargs:
216 # Already standardized to exactly what we want.
217 return mapping
218 elif kwargs.keys().isdisjoint(graph.dimensions.names):
219 # User provided kwargs, but told us not to use them by
220 # passing in dimensions that are disjoint from those kwargs.
221 # This is not necessarily user error - it's a useful pattern
222 # to pass in all of the key-value pairs you have and let the
223 # code here pull out only what it needs.
224 return mapping.subset(graph)
225 assert universe is None or universe == mapping.universe
226 universe = mapping.universe
227 d.update((name, mapping[name]) for name in mapping.graph.required.names)
228 if mapping.hasFull():
229 d.update((name, mapping[name]) for name in mapping.graph.implied.names)
230 elif isinstance(mapping, NamedKeyMapping):
231 d.update(mapping.byName())
232 elif mapping is not None:
233 d.update(mapping)
234 d.update(kwargs)
235 if graph is None:
236 if defaults is not None:
237 universe = defaults.universe
238 elif universe is None:
239 raise TypeError("universe must be provided if graph is not.")
240 graph = DimensionGraph(universe, names=d.keys())
241 if not graph.dimensions:
242 return DataCoordinate.makeEmpty(graph.universe)
243 if defaults is not None:
244 if defaults.hasFull():
245 for k, v in defaults.full.items():
246 d.setdefault(k.name, v)
247 else:
248 for k, v in defaults.items():
249 d.setdefault(k.name, v)
250 if d.keys() >= graph.dimensions.names:
251 values = tuple(d[name] for name in graph._dataCoordinateIndices)
252 else:
253 try:
254 values = tuple(d[name] for name in graph.required.names)
255 except KeyError as err:
256 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err
257 # Some backends cannot handle numpy.int64 type which is a subclass of
258 # numbers.Integral; convert that to int.
259 values = tuple(
260 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore
261 )
262 return _BasicTupleDataCoordinate(graph, values)
264 @staticmethod
265 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate:
266 """Return an empty `DataCoordinate`.
268 It identifies the null set of dimensions.
270 Parameters
271 ----------
272 universe : `DimensionUniverse`
273 Universe to which this null dimension set belongs.
275 Returns
276 -------
277 dataId : `DataCoordinate`
278 A data ID object that identifies no dimensions. `hasFull` and
279 `hasRecords` are guaranteed to return `True`, because both `full`
280 and `records` are just empty mappings.
281 """
282 return _ExpandedTupleDataCoordinate(universe.empty, (), {})
284 @staticmethod
285 def fromRequiredValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate:
286 """Construct a `DataCoordinate` from required dimension values.
288 This is a low-level interface with at most assertion-level checking of
289 inputs. Most callers should use `standardize` instead.
291 Parameters
292 ----------
293 graph : `DimensionGraph`
294 Dimensions this data ID will identify.
295 values : `tuple` [ `int` or `str` ]
296 Tuple of primary key values corresponding to ``graph.required``,
297 in that order.
299 Returns
300 -------
301 dataId : `DataCoordinate`
302 A data ID object that identifies the given dimensions.
303 ``dataId.hasFull()`` will return `True` if and only if
304 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never
305 return `True`.
306 """
307 assert len(graph.required) == len(
308 values
309 ), f"Inconsistency between dimensions {graph.required} and required values {values}."
310 return _BasicTupleDataCoordinate(graph, values)
312 @staticmethod
313 def fromFullValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate:
314 """Construct a `DataCoordinate` from all dimension values.
316 This is a low-level interface with at most assertion-level checking of
317 inputs. Most callers should use `standardize` instead.
319 Parameters
320 ----------
321 graph : `DimensionGraph`
322 Dimensions this data ID will identify.
323 values : `tuple` [ `int` or `str` ]
324 Tuple of primary key values corresponding to
325 ``itertools.chain(graph.required, graph.implied)``, in that order.
326 Note that this is _not_ the same order as ``graph.dimensions``,
327 though these contain the same elements.
329 Returns
330 -------
331 dataId : `DataCoordinate`
332 A data ID object that identifies the given dimensions.
333 ``dataId.hasFull()`` will return `True` if and only if
334 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never
335 return `True`.
336 """
337 assert len(graph.dimensions) == len(
338 values
339 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}."
340 return _BasicTupleDataCoordinate(graph, values)
342 def __hash__(self) -> int:
343 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required))
345 def __eq__(self, other: Any) -> bool:
346 if not isinstance(other, DataCoordinate):
347 other = DataCoordinate.standardize(other, universe=self.universe)
348 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required)
350 def __repr__(self) -> str:
351 # We can't make repr yield something that could be exec'd here without
352 # printing out the whole DimensionUniverse the graph is derived from.
353 # So we print something that mostly looks like a dict, but doesn't
354 # quote its keys: that's both more compact and something that can't
355 # be mistaken for an actual dict or something that could be exec'd.
356 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names]
357 if self.hasFull() and self.graph.required != self.graph.dimensions:
358 terms.append("...")
359 return "{{{}}}".format(", ".join(terms))
361 def __lt__(self, other: Any) -> bool:
362 # Allow DataCoordinate to be sorted
363 if not isinstance(other, type(self)):
364 return NotImplemented
365 # Form tuple of tuples for each DataCoordinate:
366 # Unlike repr() we only use required keys here to ensure that
367 # __eq__ can not be true simultaneously with __lt__ being true.
368 self_kv = tuple(self.items())
369 other_kv = tuple(other.items())
371 return self_kv < other_kv
373 def __iter__(self) -> Iterator[Dimension]:
374 return iter(self.keys())
376 def __len__(self) -> int:
377 return len(self.keys())
379 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore
380 return self.graph.required
382 @property
383 def names(self) -> Set[str]:
384 """Names of the required dimensions identified by this data ID.
386 They are returned in the same order as `keys`
387 (`collections.abc.Set` [ `str` ]).
388 """
389 return self.keys().names
391 @abstractmethod
392 def subset(self, graph: DimensionGraph) -> DataCoordinate:
393 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``.
395 Parameters
396 ----------
397 graph : `DimensionGraph`
398 The dimensions identified by the returned `DataCoordinate`.
400 Returns
401 -------
402 coordinate : `DataCoordinate`
403 A `DataCoordinate` instance that identifies only the given
404 dimensions. May be ``self`` if ``graph == self.graph``.
406 Raises
407 ------
408 KeyError
409 Raised if the primary key value for one or more required dimensions
410 is unknown. This may happen if ``graph.issubset(self.graph)`` is
411 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if
412 ``self.hasFull()`` is `False` and
413 ``graph.required.issubset(self.graph.required)`` is `False`. As
414 an example of the latter case, consider trying to go from a data ID
415 with dimensions {instrument, physical_filter, band} to
416 just {instrument, band}; band is implied by
417 physical_filter and hence would have no value in the original data
418 ID if ``self.hasFull()`` is `False`.
420 Notes
421 -----
422 If `hasFull` and `hasRecords` return `True` on ``self``, they will
423 return `True` (respectively) on the returned `DataCoordinate` as well.
424 The converse does not hold.
425 """
426 raise NotImplementedError()
428 @abstractmethod
429 def union(self, other: DataCoordinate) -> DataCoordinate:
430 """Combine two data IDs.
432 Yields a new one that identifies all dimensions that either of them
433 identify.
435 Parameters
436 ----------
437 other : `DataCoordinate`
438 Data ID to combine with ``self``.
440 Returns
441 -------
442 unioned : `DataCoordinate`
443 A `DataCoordinate` instance that satisfies
444 ``unioned.graph == self.graph.union(other.graph)``. Will preserve
445 ``hasFull`` and ``hasRecords`` whenever possible.
447 Notes
448 -----
449 No checking for consistency is performed on values for keys that
450 ``self`` and ``other`` have in common, and which value is included in
451 the returned data ID is not specified.
452 """
453 raise NotImplementedError()
455 @abstractmethod
456 def expanded(
457 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None]
458 ) -> DataCoordinate:
459 """Return a `DataCoordinate` that holds the given records.
461 Guarantees that `hasRecords` returns `True`.
463 This is a low-level interface with at most assertion-level checking of
464 inputs. Most callers should use `Registry.expandDataId` instead.
466 Parameters
467 ----------
468 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or \
469 `None` ]
470 A `NamedKeyMapping` with `DimensionElement` keys or a regular
471 `~collections.abc.Mapping` with `str` (`DimensionElement` name)
472 keys and `DimensionRecord` values. Keys must cover all elements in
473 ``self.graph.elements``. Values may be `None`, but only to reflect
474 actual NULL values in the database, not just records that have not
475 been fetched.
476 """
477 raise NotImplementedError()
479 @property
480 def universe(self) -> DimensionUniverse:
481 """Universe that defines all known compatible dimensions.
483 The univers will be compatible with this coordinate
484 (`DimensionUniverse`).
485 """
486 return self.graph.universe
488 @property
489 @abstractmethod
490 def graph(self) -> DimensionGraph:
491 """Dimensions identified by this data ID (`DimensionGraph`).
493 Note that values are only required to be present for dimensions in
494 ``self.graph.required``; all others may be retrieved (from a
495 `Registry`) given these.
496 """
497 raise NotImplementedError()
499 @abstractmethod
500 def hasFull(self) -> bool:
501 """Whether this data ID contains implied and required values.
503 Returns
504 -------
505 state : `bool`
506 If `True`, `__getitem__`, `get`, and `__contains__` (but not
507 `keys`!) will act as though the mapping includes key-value pairs
508 for implied dimensions, and the `full` property may be used. If
509 `False`, these operations only include key-value pairs for required
510 dimensions, and accessing `full` is an error. Always `True` if
511 there are no implied dimensions.
512 """
513 raise NotImplementedError()
515 @property
516 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]:
517 """Return mapping for all dimensions in ``self.graph``.
519 The mapping includes key-value pairs for all dimensions in
520 ``self.graph``, including implied (`NamedKeyMapping`).
522 Accessing this attribute if `hasFull` returns `False` is a logic error
523 that may raise an exception of unspecified type either immediately or
524 when implied keys are accessed via the returned mapping, depending on
525 the implementation and whether assertions are enabled.
526 """
527 assert self.hasFull(), "full may only be accessed if hasFull() returns True."
528 return _DataCoordinateFullView(self)
530 @abstractmethod
531 def hasRecords(self) -> bool:
532 """Whether this data ID contains records.
534 These are the records for all of the dimension elements it identifies.
536 Returns
537 -------
538 state : `bool`
539 If `True`, the following attributes may be accessed:
541 - `records`
542 - `region`
543 - `timespan`
544 - `pack`
546 If `False`, accessing any of these is considered a logic error.
547 """
548 raise NotImplementedError()
550 @property
551 def records(self) -> NamedKeyMapping[DimensionElement, DimensionRecord | None]:
552 """Return the records.
554 Returns a mapping that contains `DimensionRecord` objects for all
555 elements identified by this data ID (`NamedKeyMapping`).
557 The values of this mapping may be `None` if and only if there is no
558 record for that element with these dimensions in the database (which
559 means some foreign key field must have a NULL value).
561 Accessing this attribute if `hasRecords` returns `False` is a logic
562 error that may raise an exception of unspecified type either
563 immediately or when the returned mapping is used, depending on the
564 implementation and whether assertions are enabled.
565 """
566 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True."
567 return _DataCoordinateRecordsView(self)
569 @abstractmethod
570 def _record(self, name: str) -> DimensionRecord | None:
571 """Protected implementation hook that backs the ``records`` attribute.
573 Parameters
574 ----------
575 name : `str`
576 The name of a `DimensionElement`, guaranteed to be in
577 ``self.graph.elements.names``.
579 Returns
580 -------
581 record : `DimensionRecord` or `None`
582 The dimension record for the given element identified by this
583 data ID, or `None` if there is no such record.
584 """
585 raise NotImplementedError()
587 @property
588 def region(self) -> Region | None:
589 """Spatial region associated with this data ID.
591 (`lsst.sphgeom.Region` or `None`).
593 This is `None` if and only if ``self.graph.spatial`` is empty.
595 Accessing this attribute if `hasRecords` returns `False` is a logic
596 error that may or may not raise an exception, depending on the
597 implementation and whether assertions are enabled.
598 """
599 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True."
600 regions = []
601 for family in self.graph.spatial:
602 element = family.choose(self.graph.elements)
603 record = self._record(element.name)
604 if record is None or record.region is None:
605 return None
606 else:
607 regions.append(record.region)
608 return _intersectRegions(*regions)
610 @property
611 def timespan(self) -> Timespan | None:
612 """Temporal interval associated with this data ID.
614 (`Timespan` or `None`).
616 This is `None` if and only if ``self.graph.timespan`` is empty.
618 Accessing this attribute if `hasRecords` returns `False` is a logic
619 error that may or may not raise an exception, depending on the
620 implementation and whether assertions are enabled.
621 """
622 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True."
623 timespans = []
624 for family in self.graph.temporal:
625 element = family.choose(self.graph.elements)
626 record = self._record(element.name)
627 # DimensionRecord subclasses for temporal elements always have
628 # .timespan, but they're dynamic so this can't be type-checked.
629 if record is None or record.timespan is None:
630 return None
631 else:
632 timespans.append(record.timespan)
633 if not timespans:
634 return None
635 elif len(timespans) == 1:
636 return timespans[0]
637 else:
638 return Timespan.intersection(*timespans)
640 @overload
641 def pack(self, name: str, *, returnMaxBits: Literal[True]) -> tuple[int, int]:
642 ...
644 @overload
645 def pack(self, name: str, *, returnMaxBits: Literal[False]) -> int:
646 ...
648 # TODO: Remove this method and its overloads above on DM-38687.
649 @deprecated(
650 "Deprecated in favor of configurable dimension packers. Will be removed after v26.",
651 version="v26",
652 category=FutureWarning,
653 )
654 def pack(self, name: str, *, returnMaxBits: bool = False) -> tuple[int, int] | int:
655 """Pack this data ID into an integer.
657 Parameters
658 ----------
659 name : `str`
660 Name of the `DimensionPacker` algorithm (as defined in the
661 dimension configuration).
662 returnMaxBits : `bool`, optional
663 If `True` (`False` is default), return the maximum number of
664 nonzero bits in the returned integer across all data IDs.
666 Returns
667 -------
668 packed : `int`
669 Integer ID. This ID is unique only across data IDs that have
670 the same values for the packer's "fixed" dimensions.
671 maxBits : `int`, optional
672 Maximum number of nonzero bits in ``packed``. Not returned unless
673 ``returnMaxBits`` is `True`.
675 Notes
676 -----
677 Accessing this attribute if `hasRecords` returns `False` is a logic
678 error that may or may not raise an exception, depending on the
679 implementation and whether assertions are enabled.
680 """
681 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True."
682 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits)
684 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate:
685 """Convert this class to a simple python type.
687 This is suitable for serialization.
689 Parameters
690 ----------
691 minimal : `bool`, optional
692 Use minimal serialization. If set the records will not be attached.
694 Returns
695 -------
696 simple : `SerializedDataCoordinate`
697 The object converted to simple form.
698 """
699 # Convert to a dict form
700 if self.hasFull():
701 dataId = self.full.byName()
702 else:
703 dataId = self.byName()
704 records: dict[str, SerializedDimensionRecord] | None
705 if not minimal and self.hasRecords():
706 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None}
707 else:
708 records = None
710 return SerializedDataCoordinate(dataId=dataId, records=records)
712 @classmethod
713 def from_simple(
714 cls,
715 simple: SerializedDataCoordinate,
716 universe: DimensionUniverse | None = None,
717 registry: Registry | None = None,
718 ) -> DataCoordinate:
719 """Construct a new object from the simplified form.
721 The data is assumed to be of the form returned from the `to_simple`
722 method.
724 Parameters
725 ----------
726 simple : `dict` of [`str`, `Any`]
727 The `dict` returned by `to_simple()`.
728 universe : `DimensionUniverse`
729 The special graph of all known dimensions.
730 registry : `lsst.daf.butler.Registry`, optional
731 Registry from which a universe can be extracted. Can be `None`
732 if universe is provided explicitly.
734 Returns
735 -------
736 dataId : `DataCoordinate`
737 Newly-constructed object.
738 """
739 key = (frozenset(simple.dataId.items()), simple.records is not None)
740 cache = PersistenceContextVars.dataCoordinates.get()
741 if cache is not None and (result := cache.get(key)) is not None:
742 return result
743 if universe is None and registry is None:
744 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate")
745 if universe is None and registry is not None:
746 universe = registry.dimensions
747 if universe is None:
748 # this is for mypy
749 raise ValueError("Unable to determine a usable universe")
751 dataId = cls.standardize(simple.dataId, universe=universe)
752 if simple.records:
753 dataId = dataId.expanded(
754 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()}
755 )
756 if cache is not None:
757 cache[key] = dataId
758 return dataId
760 to_json = to_json_pydantic
761 from_json: ClassVar = classmethod(from_json_pydantic)
764DataId = DataCoordinate | Mapping[str, Any]
765"""A type-annotation alias for signatures that accept both informal data ID
766dictionaries and validated `DataCoordinate` instances.
767"""
770class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]):
771 """View class for `DataCoordinate.full`.
773 Provides the default implementation for
774 `DataCoordinate.full`.
776 Parameters
777 ----------
778 target : `DataCoordinate`
779 The `DataCoordinate` instance this object provides a view of.
780 """
782 def __init__(self, target: DataCoordinate):
783 self._target = target
785 __slots__ = ("_target",)
787 def __repr__(self) -> str:
788 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names]
789 return "{{{}}}".format(", ".join(terms))
791 def __getitem__(self, key: DataIdKey) -> DataIdValue:
792 return self._target[key]
794 def __iter__(self) -> Iterator[Dimension]:
795 return iter(self.keys())
797 def __len__(self) -> int:
798 return len(self.keys())
800 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore
801 return self._target.graph.dimensions
803 @property
804 def names(self) -> Set[str]:
805 # Docstring inherited from `NamedKeyMapping`.
806 return self.keys().names
809class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, DimensionRecord | None]):
810 """View class for `DataCoordinate.records`.
812 Provides the default implementation for
813 `DataCoordinate.records`.
815 Parameters
816 ----------
817 target : `DataCoordinate`
818 The `DataCoordinate` instance this object provides a view of.
819 """
821 def __init__(self, target: DataCoordinate):
822 self._target = target
824 __slots__ = ("_target",)
826 def __repr__(self) -> str:
827 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names]
828 return "{{{}}}".format(", ".join(terms))
830 def __str__(self) -> str:
831 return "\n".join(str(v) for v in self.values())
833 def __getitem__(self, key: DimensionElement | str) -> DimensionRecord | None:
834 if isinstance(key, DimensionElement):
835 key = key.name
836 return self._target._record(key)
838 def __iter__(self) -> Iterator[DimensionElement]:
839 return iter(self.keys())
841 def __len__(self) -> int:
842 return len(self.keys())
844 def keys(self) -> NamedValueAbstractSet[DimensionElement]: # type: ignore
845 return self._target.graph.elements
847 @property
848 def names(self) -> Set[str]:
849 # Docstring inherited from `NamedKeyMapping`.
850 return self.keys().names
853class _BasicTupleDataCoordinate(DataCoordinate):
854 """Standard implementation of `DataCoordinate`.
856 Backed by a tuple of values.
858 This class should only be accessed outside this module via the
859 `DataCoordinate` interface, and should only be constructed via the static
860 methods there.
862 Parameters
863 ----------
864 graph : `DimensionGraph`
865 The dimensions to be identified.
866 values : `tuple` [ `int` or `str` ]
867 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May
868 include values for just required dimensions (which always come first)
869 or all dimensions.
870 """
872 def __init__(self, graph: DimensionGraph, values: tuple[DataIdValue, ...]):
873 self._graph = graph
874 self._values = values
876 __slots__ = ("_graph", "_values")
878 @property
879 def graph(self) -> DimensionGraph:
880 # Docstring inherited from DataCoordinate.
881 return self._graph
883 def __getitem__(self, key: DataIdKey) -> DataIdValue:
884 # Docstring inherited from DataCoordinate.
885 if isinstance(key, Dimension):
886 key = key.name
887 index = self._graph._dataCoordinateIndices[key]
888 try:
889 return self._values[index]
890 except IndexError:
891 # Caller asked for an implied dimension, but this object only has
892 # values for the required ones.
893 raise KeyError(key) from None
895 def subset(self, graph: DimensionGraph) -> DataCoordinate:
896 # Docstring inherited from DataCoordinate.
897 if self._graph == graph:
898 return self
899 elif self.hasFull() or self._graph.required >= graph.dimensions:
900 return _BasicTupleDataCoordinate(
901 graph,
902 tuple(self[k] for k in graph._dataCoordinateIndices),
903 )
904 else:
905 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names))
907 def union(self, other: DataCoordinate) -> DataCoordinate:
908 # Docstring inherited from DataCoordinate.
909 graph = self.graph.union(other.graph)
910 # See if one or both input data IDs is already what we want to return;
911 # if so, return the most complete one we have.
912 if other.graph == graph:
913 if self.graph == graph:
914 # Input data IDs have the same graph (which is also the result
915 # graph), but may not have the same content.
916 # other might have records; self does not, so try other first.
917 # If it at least has full values, it's no worse than self.
918 if other.hasFull():
919 return other
920 else:
921 return self
922 elif other.hasFull():
923 return other
924 # There's some chance that neither self nor other has full values,
925 # but together provide enough to the union to. Let the general
926 # case below handle that.
927 elif self.graph == graph and self.hasFull():
928 # No chance at returning records. If self has full values, it's
929 # the best we can do.
930 return self
931 # General case with actual merging of dictionaries.
932 values = self.full.byName() if self.hasFull() else self.byName()
933 values.update(other.full.byName() if other.hasFull() else other.byName())
934 return DataCoordinate.standardize(values, graph=graph)
936 def expanded(
937 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None]
938 ) -> DataCoordinate:
939 # Docstring inherited from DataCoordinate
940 values = self._values
941 if not self.hasFull():
942 # Extract a complete values tuple from the attributes of the given
943 # records. It's possible for these to be inconsistent with
944 # self._values (which is a serious problem, of course), but we've
945 # documented this as a no-checking API.
946 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied)
947 return _ExpandedTupleDataCoordinate(self._graph, values, records)
949 def hasFull(self) -> bool:
950 # Docstring inherited from DataCoordinate.
951 return len(self._values) == len(self._graph._dataCoordinateIndices)
953 def hasRecords(self) -> bool:
954 # Docstring inherited from DataCoordinate.
955 return False
957 def _record(self, name: str) -> DimensionRecord | None:
958 # Docstring inherited from DataCoordinate.
959 raise AssertionError()
961 def __reduce__(self) -> tuple[Any, ...]:
962 return (_BasicTupleDataCoordinate, (self._graph, self._values))
964 def __getattr__(self, name: str) -> Any:
965 if name in self.graph.elements.names:
966 raise AttributeError(
967 f"Dimension record attribute {name!r} is only available on expanded DataCoordinates."
968 )
969 raise AttributeError(name)
972class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate):
973 """A `DataCoordinate` implementation that can hold `DimensionRecord`.
975 This class should only be accessed outside this module via the
976 `DataCoordinate` interface, and should only be constructed via calls to
977 `DataCoordinate.expanded`.
979 Parameters
980 ----------
981 graph : `DimensionGraph`
982 The dimensions to be identified.
983 values : `tuple` [ `int` or `str` ]
984 Data ID values, ordered to match ``graph._dataCoordinateIndices``.
985 May include values for just required dimensions (which always come
986 first) or all dimensions.
987 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or `None` ]
988 A `NamedKeyMapping` with `DimensionElement` keys or a regular
989 `~collections.abc.Mapping` with `str` (`DimensionElement` name) keys
990 and `DimensionRecord` values. Keys must cover all elements in
991 ``self.graph.elements``. Values may be `None`, but only to reflect
992 actual NULL values in the database, not just records that have not
993 been fetched.
994 """
996 def __init__(
997 self,
998 graph: DimensionGraph,
999 values: tuple[DataIdValue, ...],
1000 records: NameLookupMapping[DimensionElement, DimensionRecord | None],
1001 ):
1002 super().__init__(graph, values)
1003 assert super().hasFull(), "This implementation requires full dimension records."
1004 self._records = records
1006 __slots__ = ("_records",)
1008 def subset(self, graph: DimensionGraph) -> DataCoordinate:
1009 # Docstring inherited from DataCoordinate.
1010 if self._graph == graph:
1011 return self
1012 return _ExpandedTupleDataCoordinate(
1013 graph, tuple(self[k] for k in graph._dataCoordinateIndices), records=self._records
1014 )
1016 def expanded(
1017 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None]
1018 ) -> DataCoordinate:
1019 # Docstring inherited from DataCoordinate.
1020 return self
1022 def union(self, other: DataCoordinate) -> DataCoordinate:
1023 # Docstring inherited from DataCoordinate.
1024 graph = self.graph.union(other.graph)
1025 # See if one or both input data IDs is already what we want to return;
1026 # if so, return the most complete one we have.
1027 if self.graph == graph:
1028 # self has records, so even if other is also a valid result, it's
1029 # no better.
1030 return self
1031 if other.graph == graph and other.hasFull():
1032 # If other has full values, and self does not identify some of
1033 # those, it's the base we can do. It may have records, too.
1034 return other
1035 # If other does not have full values, there's a chance self may
1036 # provide the values needed to complete it. For example, self
1037 # could be {band} while other could be
1038 # {instrument, physical_filter, band}, with band unknown.
1039 # General case with actual merging of dictionaries.
1040 values = self.full.byName()
1041 values.update(other.full.byName() if other.hasFull() else other.byName())
1042 basic = DataCoordinate.standardize(values, graph=graph)
1043 # See if we can add records.
1044 if self.hasRecords() and other.hasRecords():
1045 # Sometimes the elements of a union of graphs can contain elements
1046 # that weren't in either input graph (because graph unions are only
1047 # on dimensions). e.g. {visit} | {detector} brings along
1048 # visit_detector_region.
1049 elements = set(graph.elements.names)
1050 elements -= self.graph.elements.names
1051 elements -= other.graph.elements.names
1052 if not elements:
1053 records = NamedKeyDict[DimensionElement, DimensionRecord | None](self.records)
1054 records.update(other.records)
1055 return basic.expanded(records.freeze())
1056 return basic
1058 def hasFull(self) -> bool:
1059 # Docstring inherited from DataCoordinate.
1060 return True
1062 def hasRecords(self) -> bool:
1063 # Docstring inherited from DataCoordinate.
1064 return True
1066 def _record(self, name: str) -> DimensionRecord | None:
1067 # Docstring inherited from DataCoordinate.
1068 return self._records[name]
1070 def __reduce__(self) -> tuple[Any, ...]:
1071 return (_ExpandedTupleDataCoordinate, (self._graph, self._values, self._records))
1073 def __getattr__(self, name: str) -> Any:
1074 try:
1075 return self._record(name)
1076 except KeyError:
1077 raise AttributeError(name) from None
1079 def __dir__(self) -> list[str]:
1080 result = list(super().__dir__())
1081 result.extend(self.graph.elements.names)
1082 return result