Coverage for python/lsst/daf/butler/core/dimensions/elements.py : 40%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"]
26from typing import (
27 Any,
28 AbstractSet,
29 Dict,
30 Iterable,
31 Optional,
32 Set,
33 Type,
34 TYPE_CHECKING,
35)
37from sqlalchemy import BigInteger
39from lsst.sphgeom import Pixelization
40from ..utils import immutable
41from ..named import NamedValueSet
42from .. import ddl
44if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 from .universe import DimensionUniverse
46 from .graph import DimensionGraph
47 from .records import DimensionRecord
50class RelatedDimensions:
51 """A semi-internal struct containing the names of the dimension elements
52 related to the one holding the instance of this struct.
54 This object is used as the type of `DimensionElement._related`, which is
55 considered private _to the dimensions subpackage_, rather that private or
56 protected within `DimensionElement` itself.
58 Parameters
59 ----------
60 required : `set` [ `str` ]
61 The names of other dimensions that are used to form the (compound)
62 primary key for this element, as well as foreign keys.
63 implied : `set` [ `str` ]
64 The names of other dimensions that are used to define foreign keys
65 for this element, but not primary keys.
66 spatial : `str`, optional
67 The name of a `DimensionElement` whose spatial regions this element's
68 region aggregates, or the name of this element if it has a region
69 that is not an aggregate. `None` (default) if this element is not
70 associated with region.
71 temporal : `str`, optional
72 The name of a `DimensionElement` whose timespans this element's
73 timespan aggregates, or the name of this element if it has a timespan
74 that is not an aggregate. `None` (default) if this element is not
75 associated with timespan.
76 """
77 def __init__(self, required: Set[str], implied: Set[str],
78 spatial: Optional[str] = None, temporal: Optional[str] = None):
79 self.required = required
80 self.implied = implied
81 self.spatial = spatial
82 self.temporal = temporal
83 self.dependencies = set(self.required | self.implied)
85 __slots__ = ("required", "implied", "spatial", "temporal", "dependencies")
87 def expand(self, universe: DimensionUniverse) -> None:
88 """Expand ``required`` and ``dependencies`` recursively.
90 Parameters
91 ----------
92 universe : `DimensionUniverse`
93 Object containing all other dimension elements.
94 """
95 for req in tuple(self.required):
96 other = universe.elements[req]._related
97 self.required.update(other.required)
98 self.dependencies.update(other.dependencies)
99 for dep in self.implied:
100 other = universe.elements[dep]._related
101 self.dependencies.update(other.dependencies)
103 required: Set[str]
104 """The names of dimensions that are used to form the (compound) primary key
105 for this element, as well as foreign keys.
107 For true `Dimension` instances, this should be constructed without the
108 dimension's own name, with the `Dimension` itself adding it later (after
109 `dependencies` is defined).
110 """
112 implied: Set[str]
113 """The names of other dimensions that are used to define foreign keys for
114 this element, but not primary keys.
115 """
117 dependencies: Set[str]
118 """The names of all dimensions in `required` or `implied`.
120 Immediately after construction, this is equal to the union of `required`
121 `implied`. After `expand` is called, this may not be true, as this will
122 include the required and implied dependencies (recursively) of implied
123 dependencies, while `implied` is not expanded recursively.
125 For true `Dimension` instances, this never includes the dimension's own
126 name.
127 """
129 spatial: Optional[str]
130 """The name of a dimension element to which this element delegates spatial
131 region handling, if any (see `DimensionElement.spatial`).
132 """
134 temporal: Optional[str]
135 """The name of a dimension element to which this element delegates
136 timespan handling, if any (see `DimensionElement.temporal`).
137 """
140@immutable
141class DimensionElement:
142 """A named data-organization concept that defines a label and/or metadata
143 in the dimensions system.
145 A `DimensionElement` instance typically corresponds to a table in the
146 `Registry`; the rows in that table are represented by instances of a
147 `DimensionRecord` subclass. Most `DimensionElement` instances are
148 instances of its `Dimension` subclass, which is used for elements that can
149 be used as data ID keys. The base class itself can be used for other
150 dimension tables that provide metadata keyed by true dimensions.
152 Parameters
153 ----------
154 name : `str`
155 Name of the element. Used as at least part of the table name, if
156 the dimension in associated with a database table.
157 related : `RelatedDimensions`
158 Struct containing the names of related dimensions.
159 metadata : iterable of `FieldSpec`
160 Additional metadata fields included in this element's table.
161 cached : `bool`
162 Whether `Registry` should cache records of this element in-memory.
163 viewOf : `str`, optional
164 Name of another table this element's records should be drawn from. The
165 fields of this table must be a superset of the fields of the element.
167 Notes
168 -----
169 `DimensionElement` instances should always be constructed by and retreived
170 from a `DimensionUniverse`. They are immutable after they are fully
171 constructed, and should never be copied.
173 Pickling a `DimensionElement` just records its name and universe;
174 unpickling one actually just looks up the element via the singleton
175 dictionary of all universes. This allows pickle to be used to transfer
176 elements between processes, but only when each process initializes its own
177 instance of the same `DimensionUniverse`.
178 """
180 def __init__(self, name: str, *,
181 related: RelatedDimensions,
182 metadata: Iterable[ddl.FieldSpec] = (),
183 cached: bool = False,
184 viewOf: Optional[str] = None,
185 alwaysJoin: bool = False):
186 self.name = name
187 self._related = related
188 self.metadata = NamedValueSet(metadata)
189 self.metadata.freeze()
190 self.cached = cached
191 self.viewOf = viewOf
192 self.alwaysJoin = alwaysJoin
194 def _finish(self, universe: DimensionUniverse, elementsToDo: Dict[str, DimensionElement]) -> None:
195 """Finish construction of the element and add it to the given universe.
197 For internal use by `DimensionUniverse` only.
199 Parameters
200 ----------
201 universe : `DimensionUniverse`
202 The under-construction dimension universe. It can be relied upon
203 to contain all dimensions that this element requires or implies.
204 elementsToDo : `dict` [ `str`, `DimensionElement` ]
205 A dictionary containing all dimension elements that have not yet
206 been added to ``universe``, keyed by name.
207 """
208 # Attach set self.universe and add self to universe attributes;
209 # let subclasses override which attributes by calling a separate
210 # method.
211 self._attachToUniverse(universe)
212 # Expand dependencies.
213 self._related.expand(universe)
214 # Define public DimensionElement versions of some of the name sets
215 # in self._related.
216 self.required = NamedValueSet(universe.sorted(self._related.required))
217 self.required.freeze()
218 self.implied = NamedValueSet(universe.sorted(self._related.implied))
219 self.implied.freeze()
220 # Set self.spatial and self.temporal to DimensionElement instances from
221 # the private _related versions of those.
222 for s in ("spatial", "temporal"):
223 targetName = getattr(self._related, s, None)
224 if targetName is None:
225 target = None
226 elif targetName == self.name:
227 target = self
228 else:
229 target = universe.elements.get(targetName)
230 if target is None:
231 try:
232 target = elementsToDo[targetName]
233 except KeyError as err:
234 raise LookupError(
235 f"Could not find {s} provider {targetName} for {self.name}."
236 ) from err
237 setattr(self, s, target)
238 # Attach a DimensionGraph that provides the public API for getting
239 # at requirements. Again delegate to subclasses.
240 self._attachGraph()
241 # Create and attach a DimensionRecord subclass to hold values of this
242 # dimension type.
243 from .records import _subclassDimensionRecord
244 self.RecordClass = _subclassDimensionRecord(self)
246 def _attachToUniverse(self, universe: DimensionUniverse) -> None:
247 """Add the element to the given universe.
249 Called only by `_finish`, but may be overridden by subclasses.
250 """
251 self.universe = universe
252 self.universe.elements.add(self)
254 def _attachGraph(self) -> None:
255 """Initialize the `graph` attribute for this element.
257 Called only by `_finish`, but may be overridden by subclasses.
258 """
259 from .graph import DimensionGraph
260 self.graph = DimensionGraph(self.universe, names=self._related.dependencies, conform=False)
262 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool:
263 """Return `True` if this element should be included in `DimensionGraph`
264 that includes the named dimensions.
266 For internal use by `DimensionGraph` only.
267 """
268 return self._related.required.issubset(dimensionNames)
270 def __str__(self) -> str:
271 return self.name
273 def __repr__(self) -> str:
274 return f"{type(self).__name__}({self.name})"
276 def __eq__(self, other: Any) -> bool:
277 try:
278 return self.name == other.name
279 except AttributeError:
280 return self.name == other
282 def __hash__(self) -> int:
283 return hash(self.name)
285 def __lt__(self, other: DimensionElement) -> bool:
286 try:
287 return self.universe._elementIndices[self] < self.universe._elementIndices[other]
288 except KeyError:
289 return NotImplemented
291 def __le__(self, other: DimensionElement) -> bool:
292 try:
293 return self.universe._elementIndices[self] <= self.universe._elementIndices[other]
294 except KeyError:
295 return NotImplemented
297 def __gt__(self, other: DimensionElement) -> bool:
298 try:
299 return self.universe._elementIndices[self] > self.universe._elementIndices[other]
300 except KeyError:
301 return NotImplemented
303 def __ge__(self, other: DimensionElement) -> bool:
304 try:
305 return self.universe._elementIndices[self] >= self.universe._elementIndices[other]
306 except KeyError:
307 return NotImplemented
309 def hasTable(self) -> bool:
310 """Return `True` if this element is associated with a table
311 (even if that table "belongs" to another element).
313 Instances of the `DimensionElement` base class itself are always
314 associated with tables.
315 """
316 return True
318 @classmethod
319 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement:
320 """Callable used for unpickling.
322 For internal use only.
323 """
324 return universe.elements[name]
326 def __reduce__(self) -> tuple:
327 return (self._unpickle, (self.universe, self.name))
329 # Class attributes below are shadowed by instance attributes, and are
330 # present just to hold the docstrings for those instance attributes.
332 universe: DimensionUniverse
333 """The universe of all compatible dimensions with which this element is
334 associated (`DimensionUniverse`).
335 """
337 name: str
338 """Unique name for this dimension element (`str`).
339 """
341 graph: DimensionGraph
342 """Minimal graph that includes this element (`DimensionGraph`).
344 ``self.graph.required`` includes all dimensions whose primary key values
345 are sufficient (often necessary) to uniquely identify ``self``
346 (including ``self` if ``isinstance(self, Dimension)``.
347 ``self.graph.implied`` includes all dimensions also identified (possibly
348 recursively) by this set.
349 """
351 required: NamedValueSet[Dimension]
352 """Dimensions that are sufficient (often necessary) to uniquely identify
353 a record of this dimension element.
355 For elements with a database representation, these dimension are exactly
356 those used to form the (possibly compound) primary key, and all dimensions
357 here that are not ``self`` are also used to form foreign keys.
359 For `Dimension` instances, this should be exactly the same as
360 ``graph.required``, but that may not be true for `DimensionElement`
361 instances in general. When they differ, there are multiple combinations
362 of dimensions that uniquely identify this element, but this one is more
363 direct.
364 """
366 implied: NamedValueSet[Dimension]
367 """Other dimensions that are uniquely identified directly by a record of
368 this dimension element.
370 For elements with a database representation, these are exactly the
371 dimensions used to form foreign key constraints whose fields are not
372 (wholly) also part of the primary key.
374 Unlike ``self.graph.implied``, this set is not expanded recursively.
375 """
377 spatial: Optional[DimensionElement]
378 """A `DimensionElement` whose spatial regions this element's region
379 aggregates, or ``self`` if it has a region that is not an aggregate
380 (`DimensionElement` or `None`).
381 """
383 temporal: Optional[DimensionElement]
384 """A `DimensionElement` whose timespans this element's timespan
385 aggregates, or ``self`` if it has a timespan that is not an aggregate
386 (`DimensionElement` or `None`).
387 """
389 metadata: NamedValueSet[ddl.FieldSpec]
390 """Additional metadata fields included in this element's table
391 (`NamedValueSet` of `FieldSpec`).
392 """
394 RecordClass: Type[DimensionRecord]
395 """The `DimensionRecord` subclass used to hold records for this element
396 (`type`).
398 Because `DimensionRecord` subclasses are generated dynamically, this type
399 cannot be imported directly and hence canonly be obtained from this
400 attribute.
401 """
403 cached: bool
404 """Whether `Registry` should cache records of this element in-memory
405 (`bool`).
406 """
408 viewOf: Optional[str]
409 """Name of another table this elements records are drawn from (`str` or
410 `None`).
411 """
414@immutable
415class Dimension(DimensionElement):
416 """A named data-organization concept that can be used as a key in a data
417 ID.
419 Parameters
420 ----------
421 name : `str`
422 Name of the dimension. Used as at least part of the table name, if
423 the dimension in associated with a database table, and as an alternate
424 key (instead of the instance itself) in data IDs.
425 related : `RelatedDimensions`
426 Struct containing the names of related dimensions.
427 uniqueKeys : iterable of `FieldSpec`
428 Fields that can *each* be used to uniquely identify this dimension
429 (once all required dependencies are also identified). The first entry
430 will be used as the table's primary key and as a foriegn key field in
431 the fields of dependent tables.
432 kwds
433 Additional keyword arguments are forwarded to the `DimensionElement`
434 constructor.
435 """
437 def __init__(self, name: str, *, related: RelatedDimensions, uniqueKeys: Iterable[ddl.FieldSpec],
438 **kwargs: Any):
439 related.required.add(name)
440 super().__init__(name, related=related, **kwargs)
441 self.uniqueKeys = NamedValueSet(uniqueKeys)
442 self.uniqueKeys.freeze()
443 self.primaryKey, *alternateKeys = uniqueKeys
444 self.alternateKeys = NamedValueSet(alternateKeys)
445 self.alternateKeys.freeze()
447 def _attachToUniverse(self, universe: DimensionUniverse) -> None:
448 # Docstring inherited from DimensionElement._attachToUniverse.
449 super()._attachToUniverse(universe)
450 universe.dimensions.add(self)
452 def _attachGraph(self) -> None:
453 # Docstring inherited from DimensionElement._attachGraph.
454 from .graph import DimensionGraph
455 self.graph = DimensionGraph(self.universe,
456 names=self._related.dependencies.union([self.name]),
457 conform=False)
459 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool:
460 # Docstring inherited from DimensionElement._shouldBeInGraph.
461 return self.name in dimensionNames
463 # Class attributes below are shadowed by instance attributes, and are
464 # present just to hold the docstrings for those instance attributes.
466 uniqueKeys: NamedValueSet[ddl.FieldSpec]
467 """All fields that can individually be used to identify records of this
468 element, given the primary keys of all required dependencies
469 (`NamedValueSet` of `FieldSpec`).
470 """
472 primaryKey: ddl.FieldSpec
473 """The primary key field for this dimension (`FieldSpec`).
475 Note that the database primary keys for dimension tables are in general
476 compound; this field is the only field in the database primary key that is
477 not also a foreign key (to a required dependency dimension table).
478 """
480 alternateKeys: NamedValueSet[ddl.FieldSpec]
481 """Additional unique key fields for this dimension that are not the the
482 primary key (`NamedValueSet` of `FieldSpec`).
483 """
486@immutable
487class SkyPixDimension(Dimension):
488 """A special `Dimension` subclass for hierarchical pixelizations of the
489 sky.
491 Unlike most other dimensions, skypix dimension records are not stored in
492 the database, as these records only contain an integer pixel ID and a
493 region on the sky, and each of these can be computed directly from the
494 other.
496 Parameters
497 ----------
498 name : `str`
499 Name of the dimension. By convention, this is a lowercase string
500 abbreviation for the pixelization followed by its integer level,
501 such as "htm7".
502 pixelization : `sphgeom.Pixelization`
503 Pixelization instance that can compute regions from IDs and IDs from
504 points.
505 """
507 def __init__(self, name: str, pixelization: Pixelization):
508 related = RelatedDimensions(required=set(), implied=set(), spatial=name)
509 uniqueKeys = [ddl.FieldSpec(name="id", dtype=BigInteger, primaryKey=True, nullable=False)]
510 super().__init__(name, related=related, uniqueKeys=uniqueKeys)
511 self.pixelization = pixelization
513 def hasTable(self) -> bool:
514 # Docstring inherited from DimensionElement.hasTable.
515 return False
517 # Class attributes below are shadowed by instance attributes, and are
518 # present just to hold the docstrings for those instance attributes.
520 pixelization: Pixelization
521 """Pixelization instance that can compute regions from IDs and IDs from
522 points (`sphgeom.Pixelization`).
523 """