Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 28%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
23# Design notes for this module are in
24# doc/lsst.daf.butler/dev/dataCoordinate.py.
25#
27from __future__ import annotations
29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate")
31import numbers
32from abc import abstractmethod
33from typing import TYPE_CHECKING, AbstractSet, Any, Dict, Iterator, Mapping, Optional, Tuple, Union
35from lsst.sphgeom import Region
36from pydantic import BaseModel
38from ..json import from_json_pydantic, to_json_pydantic
39from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping
40from ..timespan import Timespan
41from ._elements import Dimension, DimensionElement
42from ._graph import DimensionGraph
43from ._records import DimensionRecord, SerializedDimensionRecord
45if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true
46 from ...registry import Registry
47 from ._universe import DimensionUniverse
49DataIdKey = Union[str, Dimension]
50"""Type annotation alias for the keys that can be used to index a
51DataCoordinate.
52"""
54# Pydantic will cast int to str if str is first in the Union.
55DataIdValue = Union[int, str, None]
56"""Type annotation alias for the values that can be present in a
57DataCoordinate or other data ID.
58"""
61class SerializedDataCoordinate(BaseModel):
62 """Simplified model for serializing a `DataCoordinate`."""
64 dataId: Dict[str, DataIdValue]
65 records: Optional[Dict[str, SerializedDimensionRecord]] = None
67 @classmethod
68 def direct(cls, *, dataId: Dict[str, DataIdValue], records: Dict[str, Dict]) -> SerializedDataCoordinate:
69 """Construct a `SerializedDataCoordinate` directly without validators.
71 This differs from the pydantic "construct" method in that the arguments
72 are explicitly what the model requires, and it will recurse through
73 members, constructing them from their corresponding `direct` methods.
75 This method should only be called when the inputs are trusted.
76 """
77 node = SerializedDataCoordinate.__new__(cls)
78 setter = object.__setattr__
79 setter(node, "dataId", dataId)
80 setter(
81 node,
82 "records",
83 records
84 if records is None
85 else {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()},
86 )
87 setter(node, "__fields_set__", {"dataId", "records"})
88 return node
91def _intersectRegions(*args: Region) -> Optional[Region]:
92 """Return the intersection of several regions.
94 For internal use by `ExpandedDataCoordinate` only.
96 If no regions are provided, returns `None`.
98 This is currently a placeholder; it actually returns `NotImplemented`
99 (it does *not* raise an exception) when multiple regions are given, which
100 propagates to `ExpandedDataCoordinate`. This reflects the fact that we
101 don't want to fail to construct an `ExpandedDataCoordinate` entirely when
102 we can't compute its region, and at present we don't have a high-level use
103 case for the regions of these particular data IDs.
104 """
105 if len(args) == 0:
106 return None
107 elif len(args) == 1:
108 return args[0]
109 else:
110 return NotImplemented
113class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]):
114 """Data ID dictionary.
116 An immutable data ID dictionary that guarantees that its key-value pairs
117 identify at least all required dimensions in a `DimensionGraph`.
119 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory
120 functions for private concrete implementations that should be sufficient
121 for most purposes. `standardize` is the most flexible and safe of these;
122 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are
123 more specialized and perform little or no checking of inputs.
125 Notes
126 -----
127 Like any data ID class, `DataCoordinate` behaves like a dictionary, but
128 with some subtleties:
130 - Both `Dimension` instances and `str` names thereof may be used as keys
131 in lookup operations, but iteration (and `keys`) will yield `Dimension`
132 instances. The `names` property can be used to obtain the corresponding
133 `str` names.
135 - Lookups for implied dimensions (those in ``self.graph.implied``) are
136 supported if and only if `hasFull` returns `True`, and are never
137 included in iteration or `keys`. The `full` property may be used to
138 obtain a mapping whose keys do include implied dimensions.
140 - Equality comparison with other mappings is supported, but it always
141 considers only required dimensions (as well as requiring both operands
142 to identify the same dimensions). This is not quite consistent with the
143 way mappings usually work - normally differing keys imply unequal
144 mappings - but it makes sense in this context because data IDs with the
145 same values for required dimensions but different values for implied
146 dimensions represent a serious problem with the data that
147 `DataCoordinate` cannot generally recognize on its own, and a data ID
148 that knows implied dimension values should still be able to compare as
149 equal to one that does not. This is of course not the way comparisons
150 between simple `dict` data IDs work, and hence using a `DataCoordinate`
151 instance for at least one operand in any data ID comparison is strongly
152 recommended.
153 """
155 __slots__ = ()
157 _serializedType = SerializedDataCoordinate
159 @staticmethod
160 def standardize(
161 mapping: Optional[NameLookupMapping[Dimension, DataIdValue]] = None,
162 *,
163 graph: Optional[DimensionGraph] = None,
164 universe: Optional[DimensionUniverse] = None,
165 defaults: Optional[DataCoordinate] = None,
166 **kwargs: Any,
167 ) -> DataCoordinate:
168 """Standardize the supplied dataId.
170 Adapts an arbitrary mapping and/or additional arguments into a true
171 `DataCoordinate`, or augment an existing one.
173 Parameters
174 ----------
175 mapping : `~collections.abc.Mapping`, optional
176 An informal data ID that maps dimensions or dimension names to
177 their primary key values (may also be a true `DataCoordinate`).
178 graph : `DimensionGraph`
179 The dimensions to be identified by the new `DataCoordinate`.
180 If not provided, will be inferred from the keys of ``mapping`` and
181 ``**kwargs``, and ``universe`` must be provided unless ``mapping``
182 is already a `DataCoordinate`.
183 universe : `DimensionUniverse`
184 All known dimensions and their relationships; used to expand
185 and validate dependencies when ``graph`` is not provided.
186 defaults : `DataCoordinate`, optional
187 Default dimension key-value pairs to use when needed. These are
188 never used to infer ``graph``, and are ignored if a different value
189 is provided for the same key in ``mapping`` or `**kwargs``.
190 **kwargs
191 Additional keyword arguments are treated like additional key-value
192 pairs in ``mapping``.
194 Returns
195 -------
196 coordinate : `DataCoordinate`
197 A validated `DataCoordinate` instance.
199 Raises
200 ------
201 TypeError
202 Raised if the set of optional arguments provided is not supported.
203 KeyError
204 Raised if a key-value pair for a required dimension is missing.
205 """
206 d: Dict[str, DataIdValue] = {}
207 if isinstance(mapping, DataCoordinate):
208 if graph is None:
209 if not kwargs:
210 # Already standardized to exactly what we want.
211 return mapping
212 elif kwargs.keys().isdisjoint(graph.dimensions.names):
213 # User provided kwargs, but told us not to use them by
214 # passing in dimensions that are disjoint from those kwargs.
215 # This is not necessarily user error - it's a useful pattern
216 # to pass in all of the key-value pairs you have and let the
217 # code here pull out only what it needs.
218 return mapping.subset(graph)
219 assert universe is None or universe == mapping.universe
220 universe = mapping.universe
221 d.update((name, mapping[name]) for name in mapping.graph.required.names)
222 if mapping.hasFull():
223 d.update((name, mapping[name]) for name in mapping.graph.implied.names)
224 elif isinstance(mapping, NamedKeyMapping):
225 d.update(mapping.byName())
226 elif mapping is not None:
227 d.update(mapping)
228 d.update(kwargs)
229 if graph is None:
230 if defaults is not None:
231 universe = defaults.universe
232 elif universe is None:
233 raise TypeError("universe must be provided if graph is not.")
234 graph = DimensionGraph(universe, names=d.keys())
235 if not graph.dimensions:
236 return DataCoordinate.makeEmpty(graph.universe)
237 if defaults is not None:
238 if defaults.hasFull():
239 for k, v in defaults.full.items():
240 d.setdefault(k.name, v)
241 else:
242 for k, v in defaults.items():
243 d.setdefault(k.name, v)
244 if d.keys() >= graph.dimensions.names:
245 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys())
246 else:
247 try:
248 values = tuple(d[name] for name in graph.required.names)
249 except KeyError as err:
250 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err
251 # Some backends cannot handle numpy.int64 type which is a subclass of
252 # numbers.Integral; convert that to int.
253 values = tuple(
254 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore
255 )
256 return _BasicTupleDataCoordinate(graph, values)
258 @staticmethod
259 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate:
260 """Return an empty `DataCoordinate`.
262 It identifies the null set of dimensions.
264 Parameters
265 ----------
266 universe : `DimensionUniverse`
267 Universe to which this null dimension set belongs.
269 Returns
270 -------
271 dataId : `DataCoordinate`
272 A data ID object that identifies no dimensions. `hasFull` and
273 `hasRecords` are guaranteed to return `True`, because both `full`
274 and `records` are just empty mappings.
275 """
276 return _ExpandedTupleDataCoordinate(universe.empty, (), {})
278 @staticmethod
279 def fromRequiredValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate:
280 """Construct a `DataCoordinate` from required dimension values.
282 This is a low-level interface with at most assertion-level checking of
283 inputs. Most callers should use `standardize` instead.
285 Parameters
286 ----------
287 graph : `DimensionGraph`
288 Dimensions this data ID will identify.
289 values : `tuple` [ `int` or `str` ]
290 Tuple of primary key values corresponding to ``graph.required``,
291 in that order.
293 Returns
294 -------
295 dataId : `DataCoordinate`
296 A data ID object that identifies the given dimensions.
297 ``dataId.hasFull()`` will return `True` if and only if
298 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never
299 return `True`.
300 """
301 assert len(graph.required) == len(
302 values
303 ), f"Inconsistency between dimensions {graph.required} and required values {values}."
304 return _BasicTupleDataCoordinate(graph, values)
306 @staticmethod
307 def fromFullValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate:
308 """Construct a `DataCoordinate` from all dimension values.
310 This is a low-level interface with at most assertion-level checking of
311 inputs. Most callers should use `standardize` instead.
313 Parameters
314 ----------
315 graph : `DimensionGraph`
316 Dimensions this data ID will identify.
317 values : `tuple` [ `int` or `str` ]
318 Tuple of primary key values corresponding to
319 ``itertools.chain(graph.required, graph.implied)``, in that order.
320 Note that this is _not_ the same order as ``graph.dimensions``,
321 though these contain the same elements.
323 Returns
324 -------
325 dataId : `DataCoordinate`
326 A data ID object that identifies the given dimensions.
327 ``dataId.hasFull()`` will return `True` if and only if
328 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never
329 return `True`.
330 """
331 assert len(graph.dimensions) == len(
332 values
333 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}."
334 return _BasicTupleDataCoordinate(graph, values)
336 def __hash__(self) -> int:
337 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required))
339 def __eq__(self, other: Any) -> bool:
340 if not isinstance(other, DataCoordinate):
341 other = DataCoordinate.standardize(other, universe=self.universe)
342 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required)
344 def __repr__(self) -> str:
345 # We can't make repr yield something that could be exec'd here without
346 # printing out the whole DimensionUniverse the graph is derived from.
347 # So we print something that mostly looks like a dict, but doesn't
348 # quote its keys: that's both more compact and something that can't
349 # be mistaken for an actual dict or something that could be exec'd.
350 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names]
351 if self.hasFull() and self.graph.required != self.graph.dimensions:
352 terms.append("...")
353 return "{{{}}}".format(", ".join(terms))
355 def __lt__(self, other: Any) -> bool:
356 # Allow DataCoordinate to be sorted
357 if not isinstance(other, type(self)):
358 return NotImplemented
359 # Form tuple of tuples for each DataCoordinate:
360 # Unlike repr() we only use required keys here to ensure that
361 # __eq__ can not be true simultaneously with __lt__ being true.
362 self_kv = tuple(self.items())
363 other_kv = tuple(other.items())
365 return self_kv < other_kv
367 def __iter__(self) -> Iterator[Dimension]:
368 return iter(self.keys())
370 def __len__(self) -> int:
371 return len(self.keys())
373 def keys(self) -> NamedValueAbstractSet[Dimension]:
374 return self.graph.required
376 @property
377 def names(self) -> AbstractSet[str]:
378 """Names of the required dimensions identified by this data ID.
380 They are returned in the same order as `keys`
381 (`collections.abc.Set` [ `str` ]).
382 """
383 return self.keys().names
385 @abstractmethod
386 def subset(self, graph: DimensionGraph) -> DataCoordinate:
387 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``.
389 Parameters
390 ----------
391 graph : `DimensionGraph`
392 The dimensions identified by the returned `DataCoordinate`.
394 Returns
395 -------
396 coordinate : `DataCoordinate`
397 A `DataCoordinate` instance that identifies only the given
398 dimensions. May be ``self`` if ``graph == self.graph``.
400 Raises
401 ------
402 KeyError
403 Raised if the primary key value for one or more required dimensions
404 is unknown. This may happen if ``graph.issubset(self.graph)`` is
405 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if
406 ``self.hasFull()`` is `False` and
407 ``graph.required.issubset(self.graph.required)`` is `False`. As
408 an example of the latter case, consider trying to go from a data ID
409 with dimensions {instrument, physical_filter, band} to
410 just {instrument, band}; band is implied by
411 physical_filter and hence would have no value in the original data
412 ID if ``self.hasFull()`` is `False`.
414 Notes
415 -----
416 If `hasFull` and `hasRecords` return `True` on ``self``, they will
417 return `True` (respectively) on the returned `DataCoordinate` as well.
418 The converse does not hold.
419 """
420 raise NotImplementedError()
422 @abstractmethod
423 def union(self, other: DataCoordinate) -> DataCoordinate:
424 """Combine two data IDs.
426 Yields a new one that identifies all dimensions that either of them
427 identify.
429 Parameters
430 ----------
431 other : `DataCoordinate`
432 Data ID to combine with ``self``.
434 Returns
435 -------
436 unioned : `DataCoordinate`
437 A `DataCoordinate` instance that satisfies
438 ``unioned.graph == self.graph.union(other.graph)``. Will preserve
439 ``hasFull`` and ``hasRecords`` whenever possible.
441 Notes
442 -----
443 No checking for consistency is performed on values for keys that
444 ``self`` and ``other`` have in common, and which value is included in
445 the returned data ID is not specified.
446 """
447 raise NotImplementedError()
449 @abstractmethod
450 def expanded(
451 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]]
452 ) -> DataCoordinate:
453 """Return a `DataCoordinate` that holds the given records.
455 Guarantees that `hasRecords` returns `True`.
457 This is a low-level interface with at most assertion-level checking of
458 inputs. Most callers should use `Registry.expandDataId` instead.
460 Parameters
461 ----------
462 records : `Mapping` [ `str`, `DimensionRecord` or `None` ]
463 A `NamedKeyMapping` with `DimensionElement` keys or a regular
464 `Mapping` with `str` (`DimensionElement` name) keys and
465 `DimensionRecord` values. Keys must cover all elements in
466 ``self.graph.elements``. Values may be `None`, but only to reflect
467 actual NULL values in the database, not just records that have not
468 been fetched.
469 """
470 raise NotImplementedError()
472 @property
473 def universe(self) -> DimensionUniverse:
474 """Universe that defines all known compatible dimensions.
476 The univers will be compatible with this coordinate
477 (`DimensionUniverse`).
478 """
479 return self.graph.universe
481 @property
482 @abstractmethod
483 def graph(self) -> DimensionGraph:
484 """Dimensions identified by this data ID (`DimensionGraph`).
486 Note that values are only required to be present for dimensions in
487 ``self.graph.required``; all others may be retrieved (from a
488 `Registry`) given these.
489 """
490 raise NotImplementedError()
492 @abstractmethod
493 def hasFull(self) -> bool:
494 """Whether this data ID contains implied and required values.
496 Returns
497 -------
498 state : `bool`
499 If `True`, `__getitem__`, `get`, and `__contains__` (but not
500 `keys`!) will act as though the mapping includes key-value pairs
501 for implied dimensions, and the `full` property may be used. If
502 `False`, these operations only include key-value pairs for required
503 dimensions, and accessing `full` is an error. Always `True` if
504 there are no implied dimensions.
505 """
506 raise NotImplementedError()
508 @property
509 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]:
510 """Return mapping for all dimensions in ``self.graph``.
512 The mapping includes key-value pairs for all dimensions in
513 ``self.graph``, including implied (`NamedKeyMapping`).
515 Accessing this attribute if `hasFull` returns `False` is a logic error
516 that may raise an exception of unspecified type either immediately or
517 when implied keys are accessed via the returned mapping, depending on
518 the implementation and whether assertions are enabled.
519 """
520 assert self.hasFull(), "full may only be accessed if hasFull() returns True."
521 return _DataCoordinateFullView(self)
523 @abstractmethod
524 def hasRecords(self) -> bool:
525 """Whether this data ID contains records.
527 These are the records for all of the dimension elements it identifies.
529 Returns
530 -------
531 state : `bool`
532 If `True`, the following attributes may be accessed:
534 - `records`
535 - `region`
536 - `timespan`
537 - `pack`
539 If `False`, accessing any of these is considered a logic error.
540 """
541 raise NotImplementedError()
543 @property
544 def records(self) -> NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]:
545 """Return the records.
547 Returns a mapping that contains `DimensionRecord` objects for all
548 elements identified by this data ID (`NamedKeyMapping`).
550 The values of this mapping may be `None` if and only if there is no
551 record for that element with these dimensions in the database (which
552 means some foreign key field must have a NULL value).
554 Accessing this attribute if `hasRecords` returns `False` is a logic
555 error that may raise an exception of unspecified type either
556 immediately or when the returned mapping is used, depending on the
557 implementation and whether assertions are enabled.
558 """
559 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True."
560 return _DataCoordinateRecordsView(self)
562 @abstractmethod
563 def _record(self, name: str) -> Optional[DimensionRecord]:
564 """Protected implementation hook that backs the ``records`` attribute.
566 Parameters
567 ----------
568 name : `str`
569 The name of a `DimensionElement`, guaranteed to be in
570 ``self.graph.elements.names``.
572 Returns
573 -------
574 record : `DimensionRecord` or `None`
575 The dimension record for the given element identified by this
576 data ID, or `None` if there is no such record.
577 """
578 raise NotImplementedError()
580 @property
581 def region(self) -> Optional[Region]:
582 """Spatial region associated with this data ID.
584 (`lsst.sphgeom.Region` or `None`).
586 This is `None` if and only if ``self.graph.spatial`` is empty.
588 Accessing this attribute if `hasRecords` returns `False` is a logic
589 error that may or may not raise an exception, depending on the
590 implementation and whether assertions are enabled.
591 """
592 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True."
593 regions = []
594 for family in self.graph.spatial:
595 element = family.choose(self.graph.elements)
596 record = self._record(element.name)
597 if record is None or record.region is None:
598 return None
599 else:
600 regions.append(record.region)
601 return _intersectRegions(*regions)
603 @property
604 def timespan(self) -> Optional[Timespan]:
605 """Temporal interval associated with this data ID.
607 (`Timespan` or `None`).
609 This is `None` if and only if ``self.graph.timespan`` is empty.
611 Accessing this attribute if `hasRecords` returns `False` is a logic
612 error that may or may not raise an exception, depending on the
613 implementation and whether assertions are enabled.
614 """
615 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True."
616 timespans = []
617 for family in self.graph.temporal:
618 element = family.choose(self.graph.elements)
619 record = self._record(element.name)
620 # DimensionRecord subclasses for temporal elements always have
621 # .timespan, but they're dynamic so this can't be type-checked.
622 if record is None or record.timespan is None:
623 return None
624 else:
625 timespans.append(record.timespan)
626 return Timespan.intersection(*timespans)
628 def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]:
629 """Pack this data ID into an integer.
631 Parameters
632 ----------
633 name : `str`
634 Name of the `DimensionPacker` algorithm (as defined in the
635 dimension configuration).
636 returnMaxBits : `bool`, optional
637 If `True` (`False` is default), return the maximum number of
638 nonzero bits in the returned integer across all data IDs.
640 Returns
641 -------
642 packed : `int`
643 Integer ID. This ID is unique only across data IDs that have
644 the same values for the packer's "fixed" dimensions.
645 maxBits : `int`, optional
646 Maximum number of nonzero bits in ``packed``. Not returned unless
647 ``returnMaxBits`` is `True`.
649 Notes
650 -----
651 Accessing this attribute if `hasRecords` returns `False` is a logic
652 error that may or may not raise an exception, depending on the
653 implementation and whether assertions are enabled.
654 """
655 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True."
656 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits)
658 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate:
659 """Convert this class to a simple python type.
661 This is suitable for serialization.
663 Parameters
664 ----------
665 minimal : `bool`, optional
666 Use minimal serialization. If set the records will not be attached.
668 Returns
669 -------
670 simple : `SerializedDataCoordinate`
671 The object converted to simple form.
672 """
673 # Convert to a dict form
674 if self.hasFull():
675 dataId = self.full.byName()
676 else:
677 dataId = self.byName()
678 records: Optional[Dict[str, SerializedDimensionRecord]]
679 if not minimal and self.hasRecords():
680 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None}
681 else:
682 records = None
684 return SerializedDataCoordinate(dataId=dataId, records=records)
686 @classmethod
687 def from_simple(
688 cls,
689 simple: SerializedDataCoordinate,
690 universe: Optional[DimensionUniverse] = None,
691 registry: Optional[Registry] = None,
692 ) -> DataCoordinate:
693 """Construct a new object from the simplified form.
695 The data is assumed to be of the form returned from the `to_simple`
696 method.
698 Parameters
699 ----------
700 simple : `dict` of [`str`, `Any`]
701 The `dict` returned by `to_simple()`.
702 universe : `DimensionUniverse`
703 The special graph of all known dimensions.
704 registry : `lsst.daf.butler.Registry`, optional
705 Registry from which a universe can be extracted. Can be `None`
706 if universe is provided explicitly.
708 Returns
709 -------
710 dataId : `DataCoordinate`
711 Newly-constructed object.
712 """
713 if universe is None and registry is None:
714 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate")
715 if universe is None and registry is not None:
716 universe = registry.dimensions
717 if universe is None:
718 # this is for mypy
719 raise ValueError("Unable to determine a usable universe")
721 dataId = cls.standardize(simple.dataId, universe=universe)
722 if simple.records:
723 dataId = dataId.expanded(
724 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()}
725 )
726 return dataId
728 to_json = to_json_pydantic
729 from_json = classmethod(from_json_pydantic)
732DataId = Union[DataCoordinate, Mapping[str, Any]]
733"""A type-annotation alias for signatures that accept both informal data ID
734dictionaries and validated `DataCoordinate` instances.
735"""
738class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]):
739 """View class for `DataCoordinate.full`.
741 Provides the default implementation for
742 `DataCoordinate.full`.
744 Parameters
745 ----------
746 target : `DataCoordinate`
747 The `DataCoordinate` instance this object provides a view of.
748 """
750 def __init__(self, target: DataCoordinate):
751 self._target = target
753 __slots__ = ("_target",)
755 def __repr__(self) -> str:
756 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names]
757 return "{{{}}}".format(", ".join(terms))
759 def __getitem__(self, key: DataIdKey) -> DataIdValue:
760 return self._target[key]
762 def __iter__(self) -> Iterator[Dimension]:
763 return iter(self.keys())
765 def __len__(self) -> int:
766 return len(self.keys())
768 def keys(self) -> NamedValueAbstractSet[Dimension]:
769 return self._target.graph.dimensions
771 @property
772 def names(self) -> AbstractSet[str]:
773 # Docstring inherited from `NamedKeyMapping`.
774 return self.keys().names
777class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]):
778 """View class for `DataCoordinate.records`.
780 Provides the default implementation for
781 `DataCoordinate.records`.
783 Parameters
784 ----------
785 target : `DataCoordinate`
786 The `DataCoordinate` instance this object provides a view of.
787 """
789 def __init__(self, target: DataCoordinate):
790 self._target = target
792 __slots__ = ("_target",)
794 def __repr__(self) -> str:
795 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names]
796 return "{{{}}}".format(", ".join(terms))
798 def __str__(self) -> str:
799 return "\n".join(str(v) for v in self.values())
801 def __getitem__(self, key: Union[DimensionElement, str]) -> Optional[DimensionRecord]:
802 if isinstance(key, DimensionElement):
803 key = key.name
804 return self._target._record(key)
806 def __iter__(self) -> Iterator[DimensionElement]:
807 return iter(self.keys())
809 def __len__(self) -> int:
810 return len(self.keys())
812 def keys(self) -> NamedValueAbstractSet[DimensionElement]:
813 return self._target.graph.elements
815 @property
816 def names(self) -> AbstractSet[str]:
817 # Docstring inherited from `NamedKeyMapping`.
818 return self.keys().names
821class _BasicTupleDataCoordinate(DataCoordinate):
822 """Standard implementation of `DataCoordinate`.
824 Backed by a tuple of values.
826 This class should only be accessed outside this module via the
827 `DataCoordinate` interface, and should only be constructed via the static
828 methods there.
830 Parameters
831 ----------
832 graph : `DimensionGraph`
833 The dimensions to be identified.
834 values : `tuple` [ `int` or `str` ]
835 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May
836 include values for just required dimensions (which always come first)
837 or all dimensions.
838 """
840 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...]):
841 self._graph = graph
842 self._values = values
844 __slots__ = ("_graph", "_values")
846 @property
847 def graph(self) -> DimensionGraph:
848 # Docstring inherited from DataCoordinate.
849 return self._graph
851 def __getitem__(self, key: DataIdKey) -> DataIdValue:
852 # Docstring inherited from DataCoordinate.
853 if isinstance(key, Dimension):
854 key = key.name
855 index = self._graph._dataCoordinateIndices[key]
856 try:
857 return self._values[index]
858 except IndexError:
859 # Caller asked for an implied dimension, but this object only has
860 # values for the required ones.
861 raise KeyError(key) from None
863 def subset(self, graph: DimensionGraph) -> DataCoordinate:
864 # Docstring inherited from DataCoordinate.
865 if self._graph == graph:
866 return self
867 elif self.hasFull() or self._graph.required >= graph.dimensions:
868 return _BasicTupleDataCoordinate(
869 graph,
870 tuple(self[k] for k in graph._dataCoordinateIndices.keys()),
871 )
872 else:
873 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names))
875 def union(self, other: DataCoordinate) -> DataCoordinate:
876 # Docstring inherited from DataCoordinate.
877 graph = self.graph.union(other.graph)
878 # See if one or both input data IDs is already what we want to return;
879 # if so, return the most complete one we have.
880 if other.graph == graph:
881 if self.graph == graph:
882 # Input data IDs have the same graph (which is also the result
883 # graph), but may not have the same content.
884 # other might have records; self does not, so try other first.
885 # If it at least has full values, it's no worse than self.
886 if other.hasFull():
887 return other
888 else:
889 return self
890 elif other.hasFull():
891 return other
892 # There's some chance that neither self nor other has full values,
893 # but together provide enough to the union to. Let the general
894 # case below handle that.
895 elif self.graph == graph:
896 # No chance at returning records. If self has full values, it's
897 # the best we can do.
898 if self.hasFull():
899 return self
900 # General case with actual merging of dictionaries.
901 values = self.full.byName() if self.hasFull() else self.byName()
902 values.update(other.full.byName() if other.hasFull() else other.byName())
903 return DataCoordinate.standardize(values, graph=graph)
905 def expanded(
906 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]]
907 ) -> DataCoordinate:
908 # Docstring inherited from DataCoordinate
909 values = self._values
910 if not self.hasFull():
911 # Extract a complete values tuple from the attributes of the given
912 # records. It's possible for these to be inconsistent with
913 # self._values (which is a serious problem, of course), but we've
914 # documented this as a no-checking API.
915 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied)
916 return _ExpandedTupleDataCoordinate(self._graph, values, records)
918 def hasFull(self) -> bool:
919 # Docstring inherited from DataCoordinate.
920 return len(self._values) == len(self._graph._dataCoordinateIndices)
922 def hasRecords(self) -> bool:
923 # Docstring inherited from DataCoordinate.
924 return False
926 def _record(self, name: str) -> Optional[DimensionRecord]:
927 # Docstring inherited from DataCoordinate.
928 assert False
931class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate):
932 """A `DataCoordinate` implementation that can hold `DimensionRecord`.
934 This class should only be accessed outside this module via the
935 `DataCoordinate` interface, and should only be constructed via calls to
936 `DataCoordinate.expanded`.
938 Parameters
939 ----------
940 graph : `DimensionGraph`
941 The dimensions to be identified.
942 values : `tuple` [ `int` or `str` ]
943 Data ID values, ordered to match ``graph._dataCoordinateIndices``.
944 May include values for just required dimensions (which always come
945 first) or all dimensions.
946 records : `Mapping` [ `str`, `DimensionRecord` or `None` ]
947 A `NamedKeyMapping` with `DimensionElement` keys or a regular
948 `Mapping` with `str` (`DimensionElement` name) keys and
949 `DimensionRecord` values. Keys must cover all elements in
950 ``self.graph.elements``. Values may be `None`, but only to reflect
951 actual NULL values in the database, not just records that have not
952 been fetched.
953 """
955 def __init__(
956 self,
957 graph: DimensionGraph,
958 values: Tuple[DataIdValue, ...],
959 records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]],
960 ):
961 super().__init__(graph, values)
962 assert super().hasFull(), "This implementation requires full dimension records."
963 self._records = records
965 __slots__ = ("_records",)
967 def subset(self, graph: DimensionGraph) -> DataCoordinate:
968 # Docstring inherited from DataCoordinate.
969 if self._graph == graph:
970 return self
971 return _ExpandedTupleDataCoordinate(
972 graph, tuple(self[k] for k in graph._dataCoordinateIndices.keys()), records=self._records
973 )
975 def expanded(
976 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]]
977 ) -> DataCoordinate:
978 # Docstring inherited from DataCoordinate.
979 return self
981 def union(self, other: DataCoordinate) -> DataCoordinate:
982 # Docstring inherited from DataCoordinate.
983 graph = self.graph.union(other.graph)
984 # See if one or both input data IDs is already what we want to return;
985 # if so, return the most complete one we have.
986 if self.graph == graph:
987 # self has records, so even if other is also a valid result, it's
988 # no better.
989 return self
990 if other.graph == graph:
991 # If other has full values, and self does not identify some of
992 # those, it's the base we can do. It may have records, too.
993 if other.hasFull():
994 return other
995 # If other does not have full values, there's a chance self may
996 # provide the values needed to complete it. For example, self
997 # could be {band} while other could be
998 # {instrument, physical_filter, band}, with band unknown.
999 # General case with actual merging of dictionaries.
1000 values = self.full.byName()
1001 values.update(other.full.byName() if other.hasFull() else other.byName())
1002 basic = DataCoordinate.standardize(values, graph=graph)
1003 # See if we can add records.
1004 if self.hasRecords() and other.hasRecords():
1005 # Sometimes the elements of a union of graphs can contain elements
1006 # that weren't in either input graph (because graph unions are only
1007 # on dimensions). e.g. {visit} | {detector} brings along
1008 # visit_detector_region.
1009 elements = set(graph.elements.names)
1010 elements -= self.graph.elements.names
1011 elements -= other.graph.elements.names
1012 if not elements:
1013 records = NamedKeyDict[DimensionElement, Optional[DimensionRecord]](self.records)
1014 records.update(other.records)
1015 return basic.expanded(records.freeze())
1016 return basic
1018 def hasFull(self) -> bool:
1019 # Docstring inherited from DataCoordinate.
1020 return True
1022 def hasRecords(self) -> bool:
1023 # Docstring inherited from DataCoordinate.
1024 return True
1026 def _record(self, name: str) -> Optional[DimensionRecord]:
1027 # Docstring inherited from DataCoordinate.
1028 return self._records[name]