Coverage for python/lsst/daf/butler/core/dimensions/_database.py: 44%
143 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:26 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:26 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = (
25 "DatabaseDimension",
26 "DatabaseDimensionCombination",
27 "DatabaseDimensionElement",
28 "DatabaseTopologicalFamily",
29)
31from collections.abc import Iterable, Mapping, Set
32from types import MappingProxyType
33from typing import TYPE_CHECKING
35from lsst.utils import doImportType
36from lsst.utils.classes import cached_getter
38from .. import ddl
39from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace
40from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet
41from ._elements import Dimension, DimensionCombination, DimensionElement
42from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
44if TYPE_CHECKING:
45 from ...registry.interfaces import (
46 Database,
47 DatabaseDimensionRecordStorage,
48 GovernorDimensionRecordStorage,
49 StaticTablesContext,
50 )
51 from ._governor import GovernorDimension
54class DatabaseTopologicalFamily(TopologicalFamily):
55 """Database topological family implementation.
57 A `TopologicalFamily` implementation for the `DatabaseDimension` and
58 `DatabaseDimensionCombination` objects that have direct database
59 representations.
61 Parameters
62 ----------
63 name : `str`
64 Name of the family.
65 space : `TopologicalSpace`
66 Space in which this family's regions live.
67 members : `NamedValueAbstractSet` [ `DimensionElement` ]
68 The members of this family, ordered according to the priority used
69 in `choose` (first-choice member first).
70 """
72 def __init__(
73 self,
74 name: str,
75 space: TopologicalSpace,
76 *,
77 members: NamedValueAbstractSet[DimensionElement],
78 ):
79 super().__init__(name, space)
80 self.members = members
82 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement:
83 # Docstring inherited from TopologicalFamily.
84 for member in self.members:
85 if member in endpoints:
86 return member
87 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.")
89 @property
90 @cached_getter
91 def governor(self) -> GovernorDimension:
92 """Return `GovernorDimension` common to all members of this family.
94 (`GovernorDimension`).
95 """
96 governors = {m.governor for m in self.members}
97 if None in governors:
98 raise RuntimeError(
99 f"Bad {self.space.name} family definition {self.name}: at least one member "
100 f"in {self.members} has no GovernorDimension dependency."
101 )
102 try:
103 (result,) = governors
104 except ValueError:
105 raise RuntimeError(
106 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} "
107 f"in {self.members}."
108 ) from None
109 return result # type: ignore
111 members: NamedValueAbstractSet[DimensionElement]
112 """The members of this family, ordered according to the priority used in
113 `choose` (first-choice member first).
114 """
117class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor):
118 """A construction visitor for `DatabaseTopologicalFamily`.
120 This visitor depends on (and is thus visited after) its members.
122 Parameters
123 ----------
124 name : `str`
125 Name of the family.
126 members : `~collections.abc.Iterable` [ `str` ]
127 The names of the members of this family, ordered according to the
128 priority used in `choose` (first-choice member first).
129 """
131 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]):
132 super().__init__(name)
133 self._space = space
134 self._members = tuple(members)
136 def hasDependenciesIn(self, others: Set[str]) -> bool:
137 # Docstring inherited from DimensionConstructionVisitor.
138 return not others.isdisjoint(self._members)
140 def visit(self, builder: DimensionConstructionBuilder) -> None:
141 # Docstring inherited from DimensionConstructionVisitor.
142 members = NamedValueSet(builder.elements[name] for name in self._members)
143 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze())
144 builder.topology[self._space].add(family)
145 for member in members:
146 assert isinstance(member, DatabaseDimension | DatabaseDimensionCombination)
147 other = member._topology.setdefault(self._space, family)
148 if other is not family:
149 raise RuntimeError(
150 f"{member.name} is declared to be a member of (at least) two "
151 f"{self._space.name} families: {other.name} and {family.name}."
152 )
155class DatabaseDimensionElement(DimensionElement):
156 """An intermediate base class for `DimensionElement` database classes.
158 Theese classes are ones whose instances map directly to a database
159 table or query.
161 Parameters
162 ----------
163 name : `str`
164 Name of the dimension.
165 storage : `dict`
166 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
167 that will back this element in the registry (in a "cls" key) along
168 with any other construction keyword arguments (in other keys).
169 implied : `NamedValueAbstractSet` [ `Dimension` ]
170 Other dimensions whose keys are included in this dimension's (logical)
171 table as foreign keys.
172 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
173 Field specifications for all non-key fields in this dimension's table.
174 """
176 def __init__(
177 self,
178 name: str,
179 storage: dict,
180 *,
181 implied: NamedValueAbstractSet[Dimension],
182 metadata: NamedValueAbstractSet[ddl.FieldSpec],
183 ):
184 self._name = name
185 self._storage = storage
186 self._implied = implied
187 self._metadata = metadata
188 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {}
190 @property
191 def name(self) -> str:
192 # Docstring inherited from TopologicalRelationshipEndpoint.
193 return self._name
195 @property
196 def implied(self) -> NamedValueAbstractSet[Dimension]:
197 # Docstring inherited from DimensionElement.
198 return self._implied
200 @property
201 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
202 # Docstring inherited from DimensionElement.
203 return self._metadata
205 @property
206 def viewOf(self) -> str | None:
207 # Docstring inherited from DimensionElement.
208 # This is a bit encapsulation-breaking; these storage config values
209 # are supposed to be opaque here, and just forwarded on to some
210 # DimensionRecordStorage implementation. The long-term fix is to
211 # move viewOf entirely, by changing the code that relies on it to rely
212 # on the DimensionRecordStorage object instead.
213 storage = self._storage.get("nested")
214 if storage is None:
215 storage = self._storage
216 return storage.get("view_of")
218 @property
219 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]:
220 # Docstring inherited from TopologicalRelationshipEndpoint
221 return MappingProxyType(self._topology)
223 @property
224 def spatial(self) -> DatabaseTopologicalFamily | None:
225 # Docstring inherited from TopologicalRelationshipEndpoint
226 return self.topology.get(TopologicalSpace.SPATIAL)
228 @property
229 def temporal(self) -> DatabaseTopologicalFamily | None:
230 # Docstring inherited from TopologicalRelationshipEndpoint
231 return self.topology.get(TopologicalSpace.TEMPORAL)
233 def makeStorage(
234 self,
235 db: Database,
236 *,
237 context: StaticTablesContext | None = None,
238 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
239 view_target: DatabaseDimensionRecordStorage | None = None,
240 ) -> DatabaseDimensionRecordStorage:
241 """Make the dimension record storage instance for this database.
243 Constructs the `DimensionRecordStorage` instance that should
244 be used to back this element in a registry.
246 Parameters
247 ----------
248 db : `Database`
249 Interface to the underlying database engine and namespace.
250 context : `StaticTablesContext`, optional
251 If provided, an object to use to create any new tables. If not
252 provided, ``db.ensureTableExists`` should be used instead.
253 governors : `NamedKeyMapping`
254 Mapping from `GovernorDimension` to the record storage backend for
255 that dimension, containing all governor dimensions.
256 view_target : `DatabaseDimensionRecordStorage`, optional
257 Storage object for the element this target's storage is a view of
258 (i.e. when `viewOf` is not `None`).
260 Returns
261 -------
262 storage : `DatabaseDimensionRecordStorage`
263 Storage object that should back this element in a registry.
264 """
265 from ...registry.interfaces import DatabaseDimensionRecordStorage
267 cls = doImportType(self._storage["cls"])
268 assert issubclass(cls, DatabaseDimensionRecordStorage)
269 return cls.initialize(
270 db,
271 self,
272 context=context,
273 config=self._storage,
274 governors=governors,
275 view_target=view_target,
276 )
279class DatabaseDimension(Dimension, DatabaseDimensionElement):
280 """A `Dimension` class that maps directly to a database table or query.
282 Parameters
283 ----------
284 name : `str`
285 Name of the dimension.
286 storage : `dict`
287 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
288 that will back this element in the registry (in a "cls" key) along
289 with any other construction keyword arguments (in other keys).
290 required : `NamedValueSet` [ `Dimension` ]
291 Other dimensions whose keys are part of the compound primary key for
292 this dimension's (logical) table, as well as references to their own
293 tables. The ``required`` parameter does not include ``self`` (it
294 can't, of course), but the corresponding attribute does - it is added
295 by the constructor.
296 implied : `NamedValueAbstractSet` [ `Dimension` ]
297 Other dimensions whose keys are included in this dimension's (logical)
298 table as foreign keys.
299 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
300 Field specifications for all non-key fields in this dimension's table.
301 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
302 Fields that can each be used to uniquely identify this dimension (given
303 values for all required dimensions). The first of these is used as
304 (part of) this dimension's table's primary key, while others are used
305 to define unique constraints.
307 Notes
308 -----
309 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is
310 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to
311 update the `~TopologicalRelationshipEndpoint.topology` attribute of their
312 members.
313 """
315 def __init__(
316 self,
317 name: str,
318 storage: dict,
319 *,
320 required: NamedValueSet[Dimension],
321 implied: NamedValueAbstractSet[Dimension],
322 metadata: NamedValueAbstractSet[ddl.FieldSpec],
323 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec],
324 ):
325 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
326 required.add(self)
327 self._required = required.freeze()
328 self._uniqueKeys = uniqueKeys
330 @property
331 def required(self) -> NamedValueAbstractSet[Dimension]:
332 # Docstring inherited from DimensionElement.
333 return self._required
335 @property
336 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
337 # Docstring inherited from Dimension.
338 return self._uniqueKeys
341class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement):
342 """A combination class that maps directly to a database table or query.
344 Parameters
345 ----------
346 name : `str`
347 Name of the dimension.
348 storage : `dict`
349 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
350 that will back this element in the registry (in a "cls" key) along
351 with any other construction keyword arguments (in other keys).
352 required : `NamedValueAbstractSet` [ `Dimension` ]
353 Dimensions whose keys define the compound primary key for this
354 combinations's (logical) table, as well as references to their own
355 tables.
356 implied : `NamedValueAbstractSet` [ `Dimension` ]
357 Dimensions whose keys are included in this combinations's (logical)
358 table as foreign keys.
359 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
360 Field specifications for all non-key fields in this combination's
361 table.
362 alwaysJoin : `bool`, optional
363 If `True`, always include this element in any query or data ID in
364 which its ``required`` dimensions appear, because it defines a
365 relationship between those dimensions that must always be satisfied.
367 Notes
368 -----
369 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`,
370 but it is the responsibility of
371 `DatabaseTopologicalFamilyConstructionVisitor` to update the
372 `~TopologicalRelationshipEndpoint.topology` attribute of their members.
374 This class has a lot in common with `DatabaseDimension`, but they are
375 expected to diverge in future changes, and the only way to make them share
376 method implementations would be via multiple inheritance. Given the
377 trivial nature of all of those implementations, this does not seem worth
378 the drawbacks (particularly the constraints it imposes on constructor
379 signatures).
380 """
382 def __init__(
383 self,
384 name: str,
385 storage: dict,
386 *,
387 required: NamedValueAbstractSet[Dimension],
388 implied: NamedValueAbstractSet[Dimension],
389 metadata: NamedValueAbstractSet[ddl.FieldSpec],
390 alwaysJoin: bool,
391 populated_by: Dimension | None,
392 ):
393 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
394 self._required = required
395 self._alwaysJoin = alwaysJoin
396 self._populated_by = populated_by
398 @property
399 def required(self) -> NamedValueAbstractSet[Dimension]:
400 # Docstring inherited from DimensionElement.
401 return self._required
403 @property
404 def alwaysJoin(self) -> bool:
405 # Docstring inherited from DimensionElement.
406 return self._alwaysJoin
408 @property
409 def populated_by(self) -> Dimension | None:
410 # Docstring inherited.
411 return self._populated_by
414class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor):
415 """Construction visitor for database dimension and dimension combination.
417 Specifically, a construction visitor for `DatabaseDimension` and
418 `DatabaseDimensionCombination`.
420 Parameters
421 ----------
422 name : `str`
423 Name of the dimension.
424 storage : `dict`
425 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
426 that will back this element in the registry (in a "cls" key) along
427 with any other construction keyword arguments (in other keys).
428 required : `~collections.abc.Set` [ `Dimension` ]
429 Names of dimensions whose keys define the compound primary key for this
430 element's (logical) table, as well as references to their own
431 tables.
432 implied : `~collections.abc.Set` [ `Dimension` ]
433 Names of dimension whose keys are included in this elements's
434 (logical) table as foreign keys.
435 metadata : `~collections.abc.Iterable` [ `ddl.FieldSpec` ]
436 Field specifications for all non-key fields in this element's table.
437 uniqueKeys : `~collections.abc.Iterable` [ `ddl.FieldSpec` ]
438 Fields that can each be used to uniquely identify this dimension (given
439 values for all required dimensions). The first of these is used as
440 (part of) this dimension's table's primary key, while others are used
441 to define unique constraints. Should be empty for
442 `DatabaseDimensionCombination` definitions.
443 alwaysJoin : `bool`, optional
444 If `True`, always include this element in any query or data ID in
445 which its ``required`` dimensions appear, because it defines a
446 relationship between those dimensions that must always be satisfied.
447 Should only be provided when a `DimensionCombination` is being
448 constructed.
449 populated_by: `Dimension`, optional
450 The dimension that this element's records are always inserted,
451 exported, and imported alongside.
452 """
454 def __init__(
455 self,
456 name: str,
457 storage: dict,
458 required: set[str],
459 implied: set[str],
460 metadata: Iterable[ddl.FieldSpec] = (),
461 uniqueKeys: Iterable[ddl.FieldSpec] = (),
462 alwaysJoin: bool = False,
463 populated_by: str | None = None,
464 ):
465 super().__init__(name)
466 self._storage = storage
467 self._required = required
468 self._implied = implied
469 self._metadata = NamedValueSet(metadata).freeze()
470 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze()
471 self._alwaysJoin = alwaysJoin
472 self._populated_by = populated_by
474 def hasDependenciesIn(self, others: Set[str]) -> bool:
475 # Docstring inherited from DimensionConstructionVisitor.
476 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others))
478 def visit(self, builder: DimensionConstructionBuilder) -> None:
479 # Docstring inherited from DimensionConstructionVisitor.
480 # Expand required dependencies.
481 for name in tuple(self._required): # iterate over copy
482 self._required.update(builder.dimensions[name].required.names)
483 # Transform required and implied Dimension names into instances,
484 # and reorder to match builder's order.
485 required: NamedValueSet[Dimension] = NamedValueSet()
486 implied: NamedValueSet[Dimension] = NamedValueSet()
487 for dimension in builder.dimensions:
488 if dimension.name in self._required:
489 required.add(dimension)
490 if dimension.name in self._implied:
491 implied.add(dimension)
493 if self._uniqueKeys:
494 if self._alwaysJoin:
495 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.")
496 # Special handling for creating Dimension instances.
497 dimension = DatabaseDimension(
498 self.name,
499 storage=self._storage,
500 required=required,
501 implied=implied.freeze(),
502 metadata=self._metadata,
503 uniqueKeys=self._uniqueKeys,
504 )
505 builder.dimensions.add(dimension)
506 builder.elements.add(dimension)
507 else:
508 # Special handling for creating DimensionCombination instances.
509 combination = DatabaseDimensionCombination(
510 self.name,
511 storage=self._storage,
512 required=required,
513 implied=implied.freeze(),
514 metadata=self._metadata,
515 alwaysJoin=self._alwaysJoin,
516 populated_by=(
517 builder.dimensions[self._populated_by] if self._populated_by is not None else None
518 ),
519 )
520 builder.elements.add(combination)