Coverage for python/lsst/daf/butler/core/dimensions/_database.py: 34%
137 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-09 02:11 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-09 02:11 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = (
25 "DatabaseDimension",
26 "DatabaseDimensionCombination",
27 "DatabaseDimensionElement",
28 "DatabaseTopologicalFamily",
29)
31from types import MappingProxyType
32from typing import TYPE_CHECKING, AbstractSet, Dict, Iterable, Mapping, Optional, Set
34from lsst.utils import doImportType
35from lsst.utils.classes import cached_getter
37from .. import ddl
38from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace
39from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet
40from ._elements import Dimension, DimensionCombination, DimensionElement
41from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
43if TYPE_CHECKING:
44 from ...registry.interfaces import (
45 Database,
46 DatabaseDimensionRecordStorage,
47 GovernorDimensionRecordStorage,
48 StaticTablesContext,
49 )
50 from ._governor import GovernorDimension
53class DatabaseTopologicalFamily(TopologicalFamily):
54 """Database topological family implementation.
56 A `TopologicalFamily` implementation for the `DatabaseDimension` and
57 `DatabaseDimensionCombination` objects that have direct database
58 representations.
60 Parameters
61 ----------
62 name : `str`
63 Name of the family.
64 space : `TopologicalSpace`
65 Space in which this family's regions live.
66 members : `NamedValueAbstractSet` [ `DimensionElement` ]
67 The members of this family, ordered according to the priority used
68 in `choose` (first-choice member first).
69 """
71 def __init__(
72 self,
73 name: str,
74 space: TopologicalSpace,
75 *,
76 members: NamedValueAbstractSet[DimensionElement],
77 ):
78 super().__init__(name, space)
79 self.members = members
81 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement:
82 # Docstring inherited from TopologicalFamily.
83 for member in self.members:
84 if member in endpoints:
85 return member
86 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.")
88 @property
89 @cached_getter
90 def governor(self) -> GovernorDimension:
91 """Return `GovernorDimension` common to all members of this family.
93 (`GovernorDimension`).
94 """
95 governors = set(m.governor for m in self.members)
96 if None in governors:
97 raise RuntimeError(
98 f"Bad {self.space.name} family definition {self.name}: at least one member "
99 f"in {self.members} has no GovernorDimension dependency."
100 )
101 try:
102 (result,) = governors
103 except ValueError:
104 raise RuntimeError(
105 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} "
106 f"in {self.members}."
107 ) from None
108 return result # type: ignore
110 members: NamedValueAbstractSet[DimensionElement]
111 """The members of this family, ordered according to the priority used in
112 `choose` (first-choice member first).
113 """
116class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor):
117 """A construction visitor for `DatabaseTopologicalFamily`.
119 This visitor depends on (and is thus visited after) its members.
121 Parameters
122 ----------
123 name : `str`
124 Name of the family.
125 members : `Iterable` [ `str` ]
126 The names of the members of this family, ordered according to the
127 priority used in `choose` (first-choice member first).
128 """
130 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]):
131 super().__init__(name)
132 self._space = space
133 self._members = tuple(members)
135 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool:
136 # Docstring inherited from DimensionConstructionVisitor.
137 return not others.isdisjoint(self._members)
139 def visit(self, builder: DimensionConstructionBuilder) -> None:
140 # Docstring inherited from DimensionConstructionVisitor.
141 members = NamedValueSet(builder.elements[name] for name in self._members)
142 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze())
143 builder.topology[self._space].add(family)
144 for member in members:
145 assert isinstance(member, (DatabaseDimension, DatabaseDimensionCombination))
146 other = member._topology.setdefault(self._space, family)
147 if other is not family:
148 raise RuntimeError(
149 f"{member.name} is declared to be a member of (at least) two "
150 f"{self._space.name} families: {other.name} and {family.name}."
151 )
154class DatabaseDimensionElement(DimensionElement):
155 """An intermediate base class for `DimensionElement` database classes.
157 Theese classes are ones whose instances map directly to a database
158 table or query.
160 Parameters
161 ----------
162 name : `str`
163 Name of the dimension.
164 storage : `dict`
165 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
166 that will back this element in the registry (in a "cls" key) along
167 with any other construction keyword arguments (in other keys).
168 implied : `NamedValueAbstractSet` [ `Dimension` ]
169 Other dimensions whose keys are included in this dimension's (logical)
170 table as foreign keys.
171 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
172 Field specifications for all non-key fields in this dimension's table.
173 """
175 def __init__(
176 self,
177 name: str,
178 storage: dict,
179 *,
180 implied: NamedValueAbstractSet[Dimension],
181 metadata: NamedValueAbstractSet[ddl.FieldSpec],
182 ):
183 self._name = name
184 self._storage = storage
185 self._implied = implied
186 self._metadata = metadata
187 self._topology: Dict[TopologicalSpace, DatabaseTopologicalFamily] = {}
189 @property
190 def name(self) -> str:
191 # Docstring inherited from TopologicalRelationshipEndpoint.
192 return self._name
194 @property
195 def implied(self) -> NamedValueAbstractSet[Dimension]:
196 # Docstring inherited from DimensionElement.
197 return self._implied
199 @property
200 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
201 # Docstring inherited from DimensionElement.
202 return self._metadata
204 @property
205 def viewOf(self) -> Optional[str]:
206 # Docstring inherited from DimensionElement.
207 # This is a bit encapsulation-breaking; these storage config values
208 # are supposed to be opaque here, and just forwarded on to some
209 # DimensionRecordStorage implementation. The long-term fix is to
210 # move viewOf entirely, by changing the code that relies on it to rely
211 # on the DimensionRecordStorage object instead.
212 storage = self._storage.get("nested")
213 if storage is None:
214 storage = self._storage
215 return storage.get("view_of")
217 @property
218 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]:
219 # Docstring inherited from TopologicalRelationshipEndpoint
220 return MappingProxyType(self._topology)
222 @property
223 def spatial(self) -> Optional[DatabaseTopologicalFamily]:
224 # Docstring inherited from TopologicalRelationshipEndpoint
225 return self.topology.get(TopologicalSpace.SPATIAL)
227 @property
228 def temporal(self) -> Optional[DatabaseTopologicalFamily]:
229 # Docstring inherited from TopologicalRelationshipEndpoint
230 return self.topology.get(TopologicalSpace.TEMPORAL)
232 def makeStorage(
233 self,
234 db: Database,
235 *,
236 context: Optional[StaticTablesContext] = None,
237 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
238 view_target: DatabaseDimensionRecordStorage | None = None,
239 ) -> DatabaseDimensionRecordStorage:
240 """Make the dimension record storage instance for this database.
242 Constructs the `DimensionRecordStorage` instance that should
243 be used to back this element in a registry.
245 Parameters
246 ----------
247 db : `Database`
248 Interface to the underlying database engine and namespace.
249 context : `StaticTablesContext`, optional
250 If provided, an object to use to create any new tables. If not
251 provided, ``db.ensureTableExists`` should be used instead.
252 governors : `NamedKeyMapping`
253 Mapping from `GovernorDimension` to the record storage backend for
254 that dimension, containing all governor dimensions.
255 view_target : `DatabaseDimensionRecordStorage`, optional
256 Storage object for the element this target's storage is a view of
257 (i.e. when `viewOf` is not `None`).
259 Returns
260 -------
261 storage : `DatabaseDimensionRecordStorage`
262 Storage object that should back this element in a registry.
263 """
264 from ...registry.interfaces import DatabaseDimensionRecordStorage
266 cls = doImportType(self._storage["cls"])
267 assert issubclass(cls, DatabaseDimensionRecordStorage)
268 return cls.initialize(
269 db,
270 self,
271 context=context,
272 config=self._storage,
273 governors=governors,
274 view_target=view_target,
275 )
278class DatabaseDimension(Dimension, DatabaseDimensionElement):
279 """A `Dimension` class that maps directly to a database table or query.
281 Parameters
282 ----------
283 name : `str`
284 Name of the dimension.
285 storage : `dict`
286 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
287 that will back this element in the registry (in a "cls" key) along
288 with any other construction keyword arguments (in other keys).
289 required : `NamedValueSet` [ `Dimension` ]
290 Other dimensions whose keys are part of the compound primary key for
291 this dimension's (logical) table, as well as references to their own
292 tables. The ``required`` parameter does not include ``self`` (it
293 can't, of course), but the corresponding attribute does - it is added
294 by the constructor.
295 implied : `NamedValueAbstractSet` [ `Dimension` ]
296 Other dimensions whose keys are included in this dimension's (logical)
297 table as foreign keys.
298 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
299 Field specifications for all non-key fields in this dimension's table.
300 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
301 Fields that can each be used to uniquely identify this dimension (given
302 values for all required dimensions). The first of these is used as
303 (part of) this dimension's table's primary key, while others are used
304 to define unique constraints.
306 Notes
307 -----
308 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is
309 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to
310 update the `~TopologicalRelationshipEndpoint.topology` attribute of their
311 members.
312 """
314 def __init__(
315 self,
316 name: str,
317 storage: dict,
318 *,
319 required: NamedValueSet[Dimension],
320 implied: NamedValueAbstractSet[Dimension],
321 metadata: NamedValueAbstractSet[ddl.FieldSpec],
322 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec],
323 ):
324 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
325 required.add(self)
326 self._required = required.freeze()
327 self._uniqueKeys = uniqueKeys
329 @property
330 def required(self) -> NamedValueAbstractSet[Dimension]:
331 # Docstring inherited from DimensionElement.
332 return self._required
334 @property
335 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
336 # Docstring inherited from Dimension.
337 return self._uniqueKeys
340class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement):
341 """A combination class that maps directly to a database table or query.
343 Parameters
344 ----------
345 name : `str`
346 Name of the dimension.
347 storage : `dict`
348 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
349 that will back this element in the registry (in a "cls" key) along
350 with any other construction keyword arguments (in other keys).
351 required : `NamedValueAbstractSet` [ `Dimension` ]
352 Dimensions whose keys define the compound primary key for this
353 combinations's (logical) table, as well as references to their own
354 tables.
355 implied : `NamedValueAbstractSet` [ `Dimension` ]
356 Dimensions whose keys are included in this combinations's (logical)
357 table as foreign keys.
358 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
359 Field specifications for all non-key fields in this combination's
360 table.
361 alwaysJoin : `bool`, optional
362 If `True`, always include this element in any query or data ID in
363 which its ``required`` dimensions appear, because it defines a
364 relationship between those dimensions that must always be satisfied.
366 Notes
367 -----
368 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`,
369 but it is the responsibility of
370 `DatabaseTopologicalFamilyConstructionVisitor` to update the
371 `~TopologicalRelationshipEndpoint.topology` attribute of their members.
373 This class has a lot in common with `DatabaseDimension`, but they are
374 expected to diverge in future changes, and the only way to make them share
375 method implementations would be via multiple inheritance. Given the
376 trivial nature of all of those implementations, this does not seem worth
377 the drawbacks (particularly the constraints it imposes on constructor
378 signatures).
379 """
381 def __init__(
382 self,
383 name: str,
384 storage: dict,
385 *,
386 required: NamedValueAbstractSet[Dimension],
387 implied: NamedValueAbstractSet[Dimension],
388 metadata: NamedValueAbstractSet[ddl.FieldSpec],
389 alwaysJoin: bool,
390 ):
391 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
392 self._required = required
393 self._alwaysJoin = alwaysJoin
395 @property
396 def required(self) -> NamedValueAbstractSet[Dimension]:
397 # Docstring inherited from DimensionElement.
398 return self._required
400 @property
401 def alwaysJoin(self) -> bool:
402 # Docstring inherited from DimensionElement.
403 return self._alwaysJoin
406class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor):
407 """Construction visitor for database dimension and dimension combination.
409 Specifically, a construction visitor for `DatabaseDimension` and
410 `DatabaseDimensionCombination`.
412 Parameters
413 ----------
414 name : `str`
415 Name of the dimension.
416 storage : `dict`
417 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
418 that will back this element in the registry (in a "cls" key) along
419 with any other construction keyword arguments (in other keys).
420 required : `Set` [ `Dimension` ]
421 Names of dimensions whose keys define the compound primary key for this
422 element's (logical) table, as well as references to their own
423 tables.
424 implied : `Set` [ `Dimension` ]
425 Names of dimension whose keys are included in this elements's
426 (logical) table as foreign keys.
427 metadata : `Iterable` [ `ddl.FieldSpec` ]
428 Field specifications for all non-key fields in this element's table.
429 uniqueKeys : `Iterable` [ `ddl.FieldSpec` ]
430 Fields that can each be used to uniquely identify this dimension (given
431 values for all required dimensions). The first of these is used as
432 (part of) this dimension's table's primary key, while others are used
433 to define unique constraints. Should be empty for
434 `DatabaseDimensionCombination` definitions.
435 alwaysJoin : `bool`, optional
436 If `True`, always include this element in any query or data ID in
437 which its ``required`` dimensions appear, because it defines a
438 relationship between those dimensions that must always be satisfied.
439 Should only be provided when a `DimensionCombination` is being
440 constructed.
441 """
443 def __init__(
444 self,
445 name: str,
446 storage: dict,
447 required: Set[str],
448 implied: Set[str],
449 metadata: Iterable[ddl.FieldSpec] = (),
450 uniqueKeys: Iterable[ddl.FieldSpec] = (),
451 alwaysJoin: bool = False,
452 ):
453 super().__init__(name)
454 self._storage = storage
455 self._required = required
456 self._implied = implied
457 self._metadata = NamedValueSet(metadata).freeze()
458 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze()
459 self._alwaysJoin = alwaysJoin
461 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool:
462 # Docstring inherited from DimensionConstructionVisitor.
463 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others))
465 def visit(self, builder: DimensionConstructionBuilder) -> None:
466 # Docstring inherited from DimensionConstructionVisitor.
467 # Expand required dependencies.
468 for name in tuple(self._required): # iterate over copy
469 self._required.update(builder.dimensions[name].required.names)
470 # Transform required and implied Dimension names into instances,
471 # and reorder to match builder's order.
472 required: NamedValueSet[Dimension] = NamedValueSet()
473 implied: NamedValueSet[Dimension] = NamedValueSet()
474 for dimension in builder.dimensions:
475 if dimension.name in self._required:
476 required.add(dimension)
477 if dimension.name in self._implied:
478 implied.add(dimension)
480 if self._uniqueKeys:
481 if self._alwaysJoin:
482 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.")
483 # Special handling for creating Dimension instances.
484 dimension = DatabaseDimension(
485 self.name,
486 storage=self._storage,
487 required=required,
488 implied=implied.freeze(),
489 metadata=self._metadata,
490 uniqueKeys=self._uniqueKeys,
491 )
492 builder.dimensions.add(dimension)
493 builder.elements.add(dimension)
494 else:
495 # Special handling for creating DimensionCombination instances.
496 combination = DatabaseDimensionCombination(
497 self.name,
498 storage=self._storage,
499 required=required,
500 implied=implied.freeze(),
501 metadata=self._metadata,
502 alwaysJoin=self._alwaysJoin,
503 )
504 builder.elements.add(combination)