Coverage for python/lsst/daf/butler/core/dimensions/elements.py : 42%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"]
26from typing import (
27 AbstractSet,
28 Dict,
29 Iterable,
30 Optional,
31 Set,
32 TYPE_CHECKING,
33)
35from sqlalchemy import Integer
37from lsst.sphgeom import Pixelization
38from ..utils import NamedValueSet, immutable
39from .. import ddl
40from .records import _subclassDimensionRecord
41from .graph import DimensionGraph
43if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true
44 from .universe import DimensionUniverse
47class RelatedDimensions:
48 """A semi-internal struct containing the names of the dimension elements
49 related to the one holding the instance of this struct.
51 This object is used as the type of `DimensionElement._related`, which is
52 considered private _to the dimensions subpackage_, rather that private or
53 protected within `DimensionElement` itself.
55 Parameters
56 ----------
57 required : `set` [ `str` ]
58 The names of other dimensions that are used to form the (compound)
59 primary key for this element, as well as foreign keys.
60 implied : `set` [ `str` ]
61 The names of other dimensions that are used to define foreign keys
62 for this element, but not primary keys.
63 spatial : `str`, optional
64 The name of a `DimensionElement` whose spatial regions this element's
65 region aggregates, or the name of this element if it has a region
66 that is not an aggregate. `None` (default) if this element is not
67 associated with region.
68 temporal : `str`, optional
69 The name of a `DimensionElement` whose timespans this element's
70 timespan aggregates, or the name of this element if it has a timespan
71 that is not an aggregate. `None` (default) if this element is not
72 associated with timespan.
73 """
74 def __init__(self, required: Set[str], implied: Set[str],
75 spatial: Optional[str] = None, temporal: Optional[str] = None):
76 self.required = required
77 self.implied = implied
78 self.spatial = spatial
79 self.temporal = temporal
80 self.dependencies = set(self.required | self.implied)
82 __slots__ = ("required", "implied", "spatial", "temporal", "dependencies")
84 def expand(self, universe: DimensionUniverse):
85 """Expand ``required`` and ``dependencies`` recursively.
87 Parameters
88 ----------
89 universe : `DimensionUniverse`
90 Object containing all other dimension elements.
91 """
92 for req in tuple(self.required):
93 other = universe.elements[req]._related
94 self.required.update(other.required)
95 self.dependencies.update(other.dependencies)
96 for dep in self.implied:
97 other = universe.elements[dep]._related
98 self.dependencies.update(other.dependencies)
100 required: Set[str]
101 """The names of dimensions that are used to form the (compound) primary key
102 for this element, as well as foreign keys.
104 For true `Dimension` instances, this should be constructed without the
105 dimension's own name, with the `Dimension` itself adding it later (after
106 `dependencies` is defined).
107 """
109 implied: Set[str]
110 """The names of other dimensions that are used to define foreign keys for
111 this element, but not primary keys.
112 """
114 dependencies: Set[str]
115 """The names of all dimensions in `required` or `implied`.
117 Immediately after construction, this is equal to the union of `required`
118 `implied`. After `expand` is called, this may not be true, as this will
119 include the required and implied dependencies (recursively) of implied
120 dependencies, while `implied` is not expanded recursively.
122 For true `Dimension` instances, this never includes the dimension's own
123 name.
124 """
126 spatial: Optional[str]
127 """The name of a dimension element to which this element delegates spatial
128 region handling, if any (see `DimensionElement.spatial`).
129 """
131 temporal: Optional[str]
132 """The name of a dimension element to which this element delegates
133 timespan handling, if any (see `DimensionElement.temporal`).
134 """
137@immutable
138class DimensionElement:
139 """A named data-organization concept that defines a label and/or metadata
140 in the dimensions system.
142 A `DimensionElement` instance typically corresponds to a table in the
143 `Registry`; the rows in that table are represented by instances of a
144 `DimensionRecord` subclass. Most `DimensionElement` instances are
145 instances of its `Dimension` subclass, which is used for elements that can
146 be used as data ID keys. The base class itself can be used for other
147 dimension tables that provide metadata keyed by true dimensions.
149 Parameters
150 ----------
151 name : `str`
152 Name of the element. Used as at least part of the table name, if
153 the dimension in associated with a database table.
154 related : `RelatedDimensions`
155 Struct containing the names of related dimensions.
156 metadata : iterable of `FieldSpec`
157 Additional metadata fields included in this element's table.
158 cached : `bool`
159 Whether `Registry` should cache records of this element in-memory.
160 viewOf : `str`, optional
161 Name of another table this element's records should be drawn from. The
162 fields of this table must be a superset of the fields of the element.
164 Notes
165 -----
166 `DimensionElement` instances should always be constructed by and retreived
167 from a `DimensionUniverse`. They are immutable after they are fully
168 constructed, and should never be copied.
170 Pickling a `DimensionElement` just records its name and universe;
171 unpickling one actually just looks up the element via the singleton
172 dictionary of all universes. This allows pickle to be used to transfer
173 elements between processes, but only when each process initializes its own
174 instance of the same `DimensionUniverse`.
175 """
177 def __init__(self, name: str, *,
178 related: RelatedDimensions,
179 metadata: Iterable[ddl.FieldSpec] = (),
180 cached: bool = False,
181 viewOf: Optional[str] = None,
182 alwaysJoin: bool = False):
183 self.name = name
184 self._related = related
185 self.metadata = NamedValueSet(metadata)
186 self.metadata.freeze()
187 self.cached = cached
188 self.viewOf = viewOf
189 self.alwaysJoin = alwaysJoin
191 def _finish(self, universe: DimensionUniverse, elementsToDo: Dict[str, DimensionElement]):
192 """Finish construction of the element and add it to the given universe.
194 For internal use by `DimensionUniverse` only.
196 Parameters
197 ----------
198 universe : `DimensionUniverse`
199 The under-construction dimension universe. It can be relied upon
200 to contain all dimensions that this element requires or implies.
201 elementsToDo : `dict` [ `str`, `DimensionElement` ]
202 A dictionary containing all dimension elements that have not yet
203 been added to ``universe``, keyed by name.
204 """
205 # Attach set self.universe and add self to universe attributes;
206 # let subclasses override which attributes by calling a separate
207 # method.
208 self._attachToUniverse(universe)
209 # Expand dependencies.
210 self._related.expand(universe)
211 # Define public DimensionElement versions of some of the name sets
212 # in self._related.
213 self.required = NamedValueSet(universe.sorted(self._related.required))
214 self.required.freeze()
215 self.implied = NamedValueSet(universe.sorted(self._related.implied))
216 self.implied.freeze()
217 # Set self.spatial and self.temporal to DimensionElement instances from
218 # the private _related versions of those.
219 for s in ("spatial", "temporal"):
220 targetName = getattr(self._related, s, None)
221 if targetName is None:
222 target = None
223 elif targetName == self.name:
224 target = self
225 else:
226 target = universe.elements.get(targetName)
227 if target is None:
228 try:
229 target = elementsToDo[targetName]
230 except KeyError as err:
231 raise LookupError(
232 f"Could not find {s} provider {targetName} for {self.name}."
233 ) from err
234 setattr(self, s, target)
235 # Attach a DimensionGraph that provides the public API for getting
236 # at requirements. Again delegate to subclasses.
237 self._attachGraph()
238 # Create and attach a DimensionRecord subclass to hold values of this
239 # dimension type.
240 self.RecordClass = _subclassDimensionRecord(self)
242 def _attachToUniverse(self, universe: DimensionUniverse):
243 """Add the element to the given universe.
245 Called only by `_finish`, but may be overridden by subclasses.
246 """
247 self.universe = universe
248 self.universe.elements.add(self)
250 def _attachGraph(self):
251 """Initialize the `graph` attribute for this element.
253 Called only by `_finish`, but may be overridden by subclasses.
254 """
255 from .graph import DimensionGraph
256 self.graph = DimensionGraph(self.universe, names=self._related.dependencies, conform=False)
258 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]):
259 """Return `True` if this element should be included in `DimensionGraph`
260 that includes the named dimensions.
262 For internal use by `DimensionGraph` only.
263 """
264 return self._related.required.issubset(dimensionNames)
266 def __str__(self) -> str:
267 return self.name
269 def __repr__(self) -> str:
270 return f"{type(self).__name__}({self.name})"
272 def __eq__(self, other) -> bool:
273 try:
274 return self.name == other.name
275 except AttributeError:
276 return self.name == other
278 def __hash__(self) -> int:
279 return hash(self.name)
281 def __lt__(self, other) -> bool:
282 try:
283 return self.universe._elementIndices[self] < self.universe._elementIndices[other]
284 except KeyError:
285 return NotImplemented
287 def __le__(self, other) -> bool:
288 try:
289 return self.universe._elementIndices[self] <= self.universe._elementIndices[other]
290 except KeyError:
291 return NotImplemented
293 def __gt__(self, other) -> bool:
294 try:
295 return self.universe._elementIndices[self] > self.universe._elementIndices[other]
296 except KeyError:
297 return NotImplemented
299 def __ge__(self, other) -> bool:
300 try:
301 return self.universe._elementIndices[self] >= self.universe._elementIndices[other]
302 except KeyError:
303 return NotImplemented
305 def hasTable(self) -> bool:
306 """Return `True` if this element is associated with a table
307 (even if that table "belongs" to another element).
309 Instances of the `DimensionElement` base class itself are always
310 associated with tables.
311 """
312 return True
314 @classmethod
315 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement:
316 """Callable used for unpickling.
318 For internal use only.
319 """
320 return universe.elements[name]
322 def __reduce__(self) -> tuple:
323 return (self._unpickle, (self.universe, self.name))
325 # Class attributes below are shadowed by instance attributes, and are
326 # present just to hold the docstrings for those instance attributes.
328 universe: DimensionUniverse
329 """The universe of all compatible dimensions with which this element is
330 associated (`DimensionUniverse`).
331 """
333 name: str
334 """Unique name for this dimension element (`str`).
335 """
337 graph: DimensionGraph
338 """Minimal graph that includes this element (`DimensionGraph`).
340 ``self.graph.required`` includes all dimensions whose primary key values
341 are sufficient (often necessary) to uniquely identify ``self``
342 (including ``self` if ``isinstance(self, Dimension)``.
343 ``self.graph.implied`` includes all dimensions also identified (possibly
344 recursively) by this set.
345 """
347 required: NamedValueSet[Dimension]
348 """Dimensions that are sufficient (often necessary) to uniquely identify
349 a record of this dimension element.
351 For elements with a database representation, these dimension are exactly
352 those used to form the (possibly compound) primary key, and all dimensions
353 here that are not ``self`` are also used to form foreign keys.
355 For `Dimension` instances, this should be exactly the same as
356 ``graph.required``, but that may not be true for `DimensionElement`
357 instances in general. When they differ, there are multiple combinations
358 of dimensions that uniquely identify this element, but this one is more
359 direct.
360 """
362 implied: NamedValueSet[Dimension]
363 """Other dimensions that are uniquely identified directly by a record of
364 this dimension element.
366 For elements with a database representation, these are exactly the
367 dimensions used to form foreign key constraints whose fields are not
368 (wholly) also part of the primary key.
370 Unlike ``self.graph.implied``, this set is not expanded recursively.
371 """
373 spatial: Optional[DimensionElement]
374 """A `DimensionElement` whose spatial regions this element's region
375 aggregates, or ``self`` if it has a region that is not an aggregate
376 (`DimensionElement` or `None`).
377 """
379 temporal: Optional[DimensionElement]
380 """A `DimensionElement` whose timespans this element's timespan
381 aggregates, or ``self`` if it has a timespan that is not an aggregate
382 (`DimensionElement` or `None`).
383 """
385 metadata: NamedValueSet[ddl.FieldSpec]
386 """Additional metadata fields included in this element's table
387 (`NamedValueSet` of `FieldSpec`).
388 """
390 RecordClass: type
391 """The `DimensionRecord` subclass used to hold records for this element
392 (`type`).
394 Because `DimensionRecord` subclasses are generated dynamically, this type
395 cannot be imported directly and hence canonly be obtained from this
396 attribute.
397 """
399 cached: bool
400 """Whether `Registry` should cache records of this element in-memory
401 (`bool`).
402 """
404 viewOf: Optional[str]
405 """Name of another table this elements records are drawn from (`str` or
406 `None`).
407 """
410@immutable
411class Dimension(DimensionElement):
412 """A named data-organization concept that can be used as a key in a data
413 ID.
415 Parameters
416 ----------
417 name : `str`
418 Name of the dimension. Used as at least part of the table name, if
419 the dimension in associated with a database table, and as an alternate
420 key (instead of the instance itself) in data IDs.
421 related : `RelatedDimensions`
422 Struct containing the names of related dimensions.
423 uniqueKeys : iterable of `FieldSpec`
424 Fields that can *each* be used to uniquely identify this dimension
425 (once all required dependencies are also identified). The first entry
426 will be used as the table's primary key and as a foriegn key field in
427 the fields of dependent tables.
428 kwds
429 Additional keyword arguments are forwarded to the `DimensionElement`
430 constructor.
431 """
433 def __init__(self, name: str, *, related: RelatedDimensions, uniqueKeys: Iterable[ddl.FieldSpec], **kwds):
434 related.required.add(name)
435 super().__init__(name, related=related, **kwds)
436 self.uniqueKeys = NamedValueSet(uniqueKeys)
437 self.uniqueKeys.freeze()
438 self.primaryKey, *alternateKeys = uniqueKeys
439 self.alternateKeys = NamedValueSet(alternateKeys)
440 self.alternateKeys.freeze()
442 def _attachToUniverse(self, universe: DimensionUniverse):
443 # Docstring inherited from DimensionElement._attachToUniverse.
444 super()._attachToUniverse(universe)
445 universe.dimensions.add(self)
447 def _attachGraph(self):
448 # Docstring inherited from DimensionElement._attachGraph.
449 self.graph = DimensionGraph(self.universe,
450 names=self._related.dependencies.union([self.name]),
451 conform=False)
453 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]):
454 # Docstring inherited from DimensionElement._shouldBeInGraph.
455 return self.name in dimensionNames
457 # Class attributes below are shadowed by instance attributes, and are
458 # present just to hold the docstrings for those instance attributes.
460 uniqueKeys: NamedValueSet[ddl.FieldSpec]
461 """All fields that can individually be used to identify records of this
462 element, given the primary keys of all required dependencies
463 (`NamedValueSet` of `FieldSpec`).
464 """
466 primaryKey: ddl.FieldSpec
467 """The primary key field for this dimension (`FieldSpec`).
469 Note that the database primary keys for dimension tables are in general
470 compound; this field is the only field in the database primary key that is
471 not also a foreign key (to a required dependency dimension table).
472 """
474 alternateKeys: NamedValueSet[ddl.FieldSpec]
475 """Additional unique key fields for this dimension that are not the the
476 primary key (`NamedValueSet` of `FieldSpec`).
477 """
480@immutable
481class SkyPixDimension(Dimension):
482 """A special `Dimension` subclass for hierarchical pixelizations of the
483 sky.
485 Unlike most other dimensions, skypix dimension records are not stored in
486 the database, as these records only contain an integer pixel ID and a
487 region on the sky, and each of these can be computed directly from the
488 other.
490 Parameters
491 ----------
492 name : `str`
493 Name of the dimension. By convention, this is a lowercase string
494 abbreviation for the pixelization followed by its integer level,
495 such as "htm7".
496 pixelization : `sphgeom.Pixelization`
497 Pixelization instance that can compute regions from IDs and IDs from
498 points.
499 """
501 def __init__(self, name: str, pixelization: Pixelization):
502 related = RelatedDimensions(required=set(), implied=set(), spatial=self)
503 uniqueKeys = [ddl.FieldSpec(name="id", dtype=Integer, primaryKey=True, nullable=False)]
504 super().__init__(name, related=related, uniqueKeys=uniqueKeys)
505 self.pixelization = pixelization
507 def hasTable(self) -> bool:
508 # Docstring inherited from DimensionElement.hasTable.
509 return False
511 # Class attributes below are shadowed by instance attributes, and are
512 # present just to hold the docstrings for those instance attributes.
514 pixelization: Pixelization
515 """Pixelization instance that can compute regions from IDs and IDs from
516 points (`sphgeom.Pixelization`).
517 """