Coverage for python/lsst/daf/butler/core/dimensions/elements.py : 41%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"]
26from typing import (
27 Any,
28 AbstractSet,
29 Dict,
30 Iterable,
31 Optional,
32 Set,
33 TYPE_CHECKING,
34)
36from sqlalchemy import Integer
38from lsst.sphgeom import Pixelization
39from ..utils import immutable
40from ..named import NamedValueSet
41from .. import ddl
43if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true
44 from .universe import DimensionUniverse
45 from .graph import DimensionGraph
48class RelatedDimensions:
49 """A semi-internal struct containing the names of the dimension elements
50 related to the one holding the instance of this struct.
52 This object is used as the type of `DimensionElement._related`, which is
53 considered private _to the dimensions subpackage_, rather that private or
54 protected within `DimensionElement` itself.
56 Parameters
57 ----------
58 required : `set` [ `str` ]
59 The names of other dimensions that are used to form the (compound)
60 primary key for this element, as well as foreign keys.
61 implied : `set` [ `str` ]
62 The names of other dimensions that are used to define foreign keys
63 for this element, but not primary keys.
64 spatial : `str`, optional
65 The name of a `DimensionElement` whose spatial regions this element's
66 region aggregates, or the name of this element if it has a region
67 that is not an aggregate. `None` (default) if this element is not
68 associated with region.
69 temporal : `str`, optional
70 The name of a `DimensionElement` whose timespans this element's
71 timespan aggregates, or the name of this element if it has a timespan
72 that is not an aggregate. `None` (default) if this element is not
73 associated with timespan.
74 """
75 def __init__(self, required: Set[str], implied: Set[str],
76 spatial: Optional[str] = None, temporal: Optional[str] = None):
77 self.required = required
78 self.implied = implied
79 self.spatial = spatial
80 self.temporal = temporal
81 self.dependencies = set(self.required | self.implied)
83 __slots__ = ("required", "implied", "spatial", "temporal", "dependencies")
85 def expand(self, universe: DimensionUniverse) -> None:
86 """Expand ``required`` and ``dependencies`` recursively.
88 Parameters
89 ----------
90 universe : `DimensionUniverse`
91 Object containing all other dimension elements.
92 """
93 for req in tuple(self.required):
94 other = universe.elements[req]._related
95 self.required.update(other.required)
96 self.dependencies.update(other.dependencies)
97 for dep in self.implied:
98 other = universe.elements[dep]._related
99 self.dependencies.update(other.dependencies)
101 required: Set[str]
102 """The names of dimensions that are used to form the (compound) primary key
103 for this element, as well as foreign keys.
105 For true `Dimension` instances, this should be constructed without the
106 dimension's own name, with the `Dimension` itself adding it later (after
107 `dependencies` is defined).
108 """
110 implied: Set[str]
111 """The names of other dimensions that are used to define foreign keys for
112 this element, but not primary keys.
113 """
115 dependencies: Set[str]
116 """The names of all dimensions in `required` or `implied`.
118 Immediately after construction, this is equal to the union of `required`
119 `implied`. After `expand` is called, this may not be true, as this will
120 include the required and implied dependencies (recursively) of implied
121 dependencies, while `implied` is not expanded recursively.
123 For true `Dimension` instances, this never includes the dimension's own
124 name.
125 """
127 spatial: Optional[str]
128 """The name of a dimension element to which this element delegates spatial
129 region handling, if any (see `DimensionElement.spatial`).
130 """
132 temporal: Optional[str]
133 """The name of a dimension element to which this element delegates
134 timespan handling, if any (see `DimensionElement.temporal`).
135 """
138@immutable
139class DimensionElement:
140 """A named data-organization concept that defines a label and/or metadata
141 in the dimensions system.
143 A `DimensionElement` instance typically corresponds to a table in the
144 `Registry`; the rows in that table are represented by instances of a
145 `DimensionRecord` subclass. Most `DimensionElement` instances are
146 instances of its `Dimension` subclass, which is used for elements that can
147 be used as data ID keys. The base class itself can be used for other
148 dimension tables that provide metadata keyed by true dimensions.
150 Parameters
151 ----------
152 name : `str`
153 Name of the element. Used as at least part of the table name, if
154 the dimension in associated with a database table.
155 related : `RelatedDimensions`
156 Struct containing the names of related dimensions.
157 metadata : iterable of `FieldSpec`
158 Additional metadata fields included in this element's table.
159 cached : `bool`
160 Whether `Registry` should cache records of this element in-memory.
161 viewOf : `str`, optional
162 Name of another table this element's records should be drawn from. The
163 fields of this table must be a superset of the fields of the element.
165 Notes
166 -----
167 `DimensionElement` instances should always be constructed by and retreived
168 from a `DimensionUniverse`. They are immutable after they are fully
169 constructed, and should never be copied.
171 Pickling a `DimensionElement` just records its name and universe;
172 unpickling one actually just looks up the element via the singleton
173 dictionary of all universes. This allows pickle to be used to transfer
174 elements between processes, but only when each process initializes its own
175 instance of the same `DimensionUniverse`.
176 """
178 def __init__(self, name: str, *,
179 related: RelatedDimensions,
180 metadata: Iterable[ddl.FieldSpec] = (),
181 cached: bool = False,
182 viewOf: Optional[str] = None,
183 alwaysJoin: bool = False):
184 self.name = name
185 self._related = related
186 self.metadata = NamedValueSet(metadata)
187 self.metadata.freeze()
188 self.cached = cached
189 self.viewOf = viewOf
190 self.alwaysJoin = alwaysJoin
192 def _finish(self, universe: DimensionUniverse, elementsToDo: Dict[str, DimensionElement]) -> None:
193 """Finish construction of the element and add it to the given universe.
195 For internal use by `DimensionUniverse` only.
197 Parameters
198 ----------
199 universe : `DimensionUniverse`
200 The under-construction dimension universe. It can be relied upon
201 to contain all dimensions that this element requires or implies.
202 elementsToDo : `dict` [ `str`, `DimensionElement` ]
203 A dictionary containing all dimension elements that have not yet
204 been added to ``universe``, keyed by name.
205 """
206 # Attach set self.universe and add self to universe attributes;
207 # let subclasses override which attributes by calling a separate
208 # method.
209 self._attachToUniverse(universe)
210 # Expand dependencies.
211 self._related.expand(universe)
212 # Define public DimensionElement versions of some of the name sets
213 # in self._related.
214 self.required = NamedValueSet(universe.sorted(self._related.required))
215 self.required.freeze()
216 self.implied = NamedValueSet(universe.sorted(self._related.implied))
217 self.implied.freeze()
218 # Set self.spatial and self.temporal to DimensionElement instances from
219 # the private _related versions of those.
220 for s in ("spatial", "temporal"):
221 targetName = getattr(self._related, s, None)
222 if targetName is None:
223 target = None
224 elif targetName == self.name:
225 target = self
226 else:
227 target = universe.elements.get(targetName)
228 if target is None:
229 try:
230 target = elementsToDo[targetName]
231 except KeyError as err:
232 raise LookupError(
233 f"Could not find {s} provider {targetName} for {self.name}."
234 ) from err
235 setattr(self, s, target)
236 # Attach a DimensionGraph that provides the public API for getting
237 # at requirements. Again delegate to subclasses.
238 self._attachGraph()
239 # Create and attach a DimensionRecord subclass to hold values of this
240 # dimension type.
241 from .records import _subclassDimensionRecord
242 self.RecordClass = _subclassDimensionRecord(self)
244 def _attachToUniverse(self, universe: DimensionUniverse) -> None:
245 """Add the element to the given universe.
247 Called only by `_finish`, but may be overridden by subclasses.
248 """
249 self.universe = universe
250 self.universe.elements.add(self)
252 def _attachGraph(self) -> None:
253 """Initialize the `graph` attribute for this element.
255 Called only by `_finish`, but may be overridden by subclasses.
256 """
257 from .graph import DimensionGraph
258 self.graph = DimensionGraph(self.universe, names=self._related.dependencies, conform=False)
260 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool:
261 """Return `True` if this element should be included in `DimensionGraph`
262 that includes the named dimensions.
264 For internal use by `DimensionGraph` only.
265 """
266 return self._related.required.issubset(dimensionNames)
268 def __str__(self) -> str:
269 return self.name
271 def __repr__(self) -> str:
272 return f"{type(self).__name__}({self.name})"
274 def __eq__(self, other: Any) -> bool:
275 try:
276 return self.name == other.name
277 except AttributeError:
278 return self.name == other
280 def __hash__(self) -> int:
281 return hash(self.name)
283 def __lt__(self, other: DimensionElement) -> bool:
284 try:
285 return self.universe._elementIndices[self] < self.universe._elementIndices[other]
286 except KeyError:
287 return NotImplemented
289 def __le__(self, other: DimensionElement) -> bool:
290 try:
291 return self.universe._elementIndices[self] <= self.universe._elementIndices[other]
292 except KeyError:
293 return NotImplemented
295 def __gt__(self, other: DimensionElement) -> bool:
296 try:
297 return self.universe._elementIndices[self] > self.universe._elementIndices[other]
298 except KeyError:
299 return NotImplemented
301 def __ge__(self, other: DimensionElement) -> bool:
302 try:
303 return self.universe._elementIndices[self] >= self.universe._elementIndices[other]
304 except KeyError:
305 return NotImplemented
307 def hasTable(self) -> bool:
308 """Return `True` if this element is associated with a table
309 (even if that table "belongs" to another element).
311 Instances of the `DimensionElement` base class itself are always
312 associated with tables.
313 """
314 return True
316 @classmethod
317 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement:
318 """Callable used for unpickling.
320 For internal use only.
321 """
322 return universe.elements[name]
324 def __reduce__(self) -> tuple:
325 return (self._unpickle, (self.universe, self.name))
327 # Class attributes below are shadowed by instance attributes, and are
328 # present just to hold the docstrings for those instance attributes.
330 universe: DimensionUniverse
331 """The universe of all compatible dimensions with which this element is
332 associated (`DimensionUniverse`).
333 """
335 name: str
336 """Unique name for this dimension element (`str`).
337 """
339 graph: DimensionGraph
340 """Minimal graph that includes this element (`DimensionGraph`).
342 ``self.graph.required`` includes all dimensions whose primary key values
343 are sufficient (often necessary) to uniquely identify ``self``
344 (including ``self` if ``isinstance(self, Dimension)``.
345 ``self.graph.implied`` includes all dimensions also identified (possibly
346 recursively) by this set.
347 """
349 required: NamedValueSet[Dimension]
350 """Dimensions that are sufficient (often necessary) to uniquely identify
351 a record of this dimension element.
353 For elements with a database representation, these dimension are exactly
354 those used to form the (possibly compound) primary key, and all dimensions
355 here that are not ``self`` are also used to form foreign keys.
357 For `Dimension` instances, this should be exactly the same as
358 ``graph.required``, but that may not be true for `DimensionElement`
359 instances in general. When they differ, there are multiple combinations
360 of dimensions that uniquely identify this element, but this one is more
361 direct.
362 """
364 implied: NamedValueSet[Dimension]
365 """Other dimensions that are uniquely identified directly by a record of
366 this dimension element.
368 For elements with a database representation, these are exactly the
369 dimensions used to form foreign key constraints whose fields are not
370 (wholly) also part of the primary key.
372 Unlike ``self.graph.implied``, this set is not expanded recursively.
373 """
375 spatial: Optional[DimensionElement]
376 """A `DimensionElement` whose spatial regions this element's region
377 aggregates, or ``self`` if it has a region that is not an aggregate
378 (`DimensionElement` or `None`).
379 """
381 temporal: Optional[DimensionElement]
382 """A `DimensionElement` whose timespans this element's timespan
383 aggregates, or ``self`` if it has a timespan that is not an aggregate
384 (`DimensionElement` or `None`).
385 """
387 metadata: NamedValueSet[ddl.FieldSpec]
388 """Additional metadata fields included in this element's table
389 (`NamedValueSet` of `FieldSpec`).
390 """
392 RecordClass: type
393 """The `DimensionRecord` subclass used to hold records for this element
394 (`type`).
396 Because `DimensionRecord` subclasses are generated dynamically, this type
397 cannot be imported directly and hence canonly be obtained from this
398 attribute.
399 """
401 cached: bool
402 """Whether `Registry` should cache records of this element in-memory
403 (`bool`).
404 """
406 viewOf: Optional[str]
407 """Name of another table this elements records are drawn from (`str` or
408 `None`).
409 """
412@immutable
413class Dimension(DimensionElement):
414 """A named data-organization concept that can be used as a key in a data
415 ID.
417 Parameters
418 ----------
419 name : `str`
420 Name of the dimension. Used as at least part of the table name, if
421 the dimension in associated with a database table, and as an alternate
422 key (instead of the instance itself) in data IDs.
423 related : `RelatedDimensions`
424 Struct containing the names of related dimensions.
425 uniqueKeys : iterable of `FieldSpec`
426 Fields that can *each* be used to uniquely identify this dimension
427 (once all required dependencies are also identified). The first entry
428 will be used as the table's primary key and as a foriegn key field in
429 the fields of dependent tables.
430 kwds
431 Additional keyword arguments are forwarded to the `DimensionElement`
432 constructor.
433 """
435 def __init__(self, name: str, *, related: RelatedDimensions, uniqueKeys: Iterable[ddl.FieldSpec],
436 **kwargs: Any):
437 related.required.add(name)
438 super().__init__(name, related=related, **kwargs)
439 self.uniqueKeys = NamedValueSet(uniqueKeys)
440 self.uniqueKeys.freeze()
441 self.primaryKey, *alternateKeys = uniqueKeys
442 self.alternateKeys = NamedValueSet(alternateKeys)
443 self.alternateKeys.freeze()
445 def _attachToUniverse(self, universe: DimensionUniverse) -> None:
446 # Docstring inherited from DimensionElement._attachToUniverse.
447 super()._attachToUniverse(universe)
448 universe.dimensions.add(self)
450 def _attachGraph(self) -> None:
451 # Docstring inherited from DimensionElement._attachGraph.
452 from .graph import DimensionGraph
453 self.graph = DimensionGraph(self.universe,
454 names=self._related.dependencies.union([self.name]),
455 conform=False)
457 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool:
458 # Docstring inherited from DimensionElement._shouldBeInGraph.
459 return self.name in dimensionNames
461 # Class attributes below are shadowed by instance attributes, and are
462 # present just to hold the docstrings for those instance attributes.
464 uniqueKeys: NamedValueSet[ddl.FieldSpec]
465 """All fields that can individually be used to identify records of this
466 element, given the primary keys of all required dependencies
467 (`NamedValueSet` of `FieldSpec`).
468 """
470 primaryKey: ddl.FieldSpec
471 """The primary key field for this dimension (`FieldSpec`).
473 Note that the database primary keys for dimension tables are in general
474 compound; this field is the only field in the database primary key that is
475 not also a foreign key (to a required dependency dimension table).
476 """
478 alternateKeys: NamedValueSet[ddl.FieldSpec]
479 """Additional unique key fields for this dimension that are not the the
480 primary key (`NamedValueSet` of `FieldSpec`).
481 """
484@immutable
485class SkyPixDimension(Dimension):
486 """A special `Dimension` subclass for hierarchical pixelizations of the
487 sky.
489 Unlike most other dimensions, skypix dimension records are not stored in
490 the database, as these records only contain an integer pixel ID and a
491 region on the sky, and each of these can be computed directly from the
492 other.
494 Parameters
495 ----------
496 name : `str`
497 Name of the dimension. By convention, this is a lowercase string
498 abbreviation for the pixelization followed by its integer level,
499 such as "htm7".
500 pixelization : `sphgeom.Pixelization`
501 Pixelization instance that can compute regions from IDs and IDs from
502 points.
503 """
505 def __init__(self, name: str, pixelization: Pixelization):
506 related = RelatedDimensions(required=set(), implied=set(), spatial=name)
507 uniqueKeys = [ddl.FieldSpec(name="id", dtype=Integer, primaryKey=True, nullable=False)]
508 super().__init__(name, related=related, uniqueKeys=uniqueKeys)
509 self.pixelization = pixelization
511 def hasTable(self) -> bool:
512 # Docstring inherited from DimensionElement.hasTable.
513 return False
515 # Class attributes below are shadowed by instance attributes, and are
516 # present just to hold the docstrings for those instance attributes.
518 pixelization: Pixelization
519 """Pixelization instance that can compute regions from IDs and IDs from
520 points (`sphgeom.Pixelization`).
521 """