Coverage for python/lsst/daf/butler/dimensions/_database.py: 44%
143 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-01 11:00 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-01 11:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = (
31 "DatabaseDimension",
32 "DatabaseDimensionCombination",
33 "DatabaseDimensionElement",
34 "DatabaseTopologicalFamily",
35)
37from collections.abc import Iterable, Mapping, Set
38from types import MappingProxyType
39from typing import TYPE_CHECKING
41from lsst.utils import doImportType
42from lsst.utils.classes import cached_getter
44from .. import ddl
45from .._named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet
46from .._topology import TopologicalFamily, TopologicalSpace
47from ._elements import Dimension, DimensionCombination, DimensionElement
48from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
50if TYPE_CHECKING:
51 from ..registry.interfaces import (
52 Database,
53 DatabaseDimensionRecordStorage,
54 GovernorDimensionRecordStorage,
55 StaticTablesContext,
56 )
57 from ._governor import GovernorDimension
58 from ._universe import DimensionUniverse
61class DatabaseTopologicalFamily(TopologicalFamily):
62 """Database topological family implementation.
64 A `TopologicalFamily` implementation for the `DatabaseDimension` and
65 `DatabaseDimensionCombination` objects that have direct database
66 representations.
68 Parameters
69 ----------
70 name : `str`
71 Name of the family.
72 space : `TopologicalSpace`
73 Space in which this family's regions live.
74 members : `NamedValueAbstractSet` [ `DimensionElement` ]
75 The members of this family, ordered according to the priority used
76 in `choose` (first-choice member first).
77 """
79 def __init__(
80 self,
81 name: str,
82 space: TopologicalSpace,
83 *,
84 members: NamedValueAbstractSet[DimensionElement],
85 ):
86 super().__init__(name, space)
87 self.members = members
89 def choose(self, endpoints: Set[str], universe: DimensionUniverse) -> DimensionElement:
90 # Docstring inherited from TopologicalFamily.
91 for member in self.members:
92 if member.name in endpoints:
93 return member
94 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.")
96 @property
97 @cached_getter
98 def governor(self) -> GovernorDimension:
99 """Return `GovernorDimension` common to all members of this family.
101 (`GovernorDimension`).
102 """
103 governors = {m.governor for m in self.members}
104 if None in governors:
105 raise RuntimeError(
106 f"Bad {self.space.name} family definition {self.name}: at least one member "
107 f"in {self.members} has no GovernorDimension dependency."
108 )
109 try:
110 (result,) = governors
111 except ValueError:
112 raise RuntimeError(
113 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} "
114 f"in {self.members}."
115 ) from None
116 return result # type: ignore
118 members: NamedValueAbstractSet[DimensionElement]
119 """The members of this family, ordered according to the priority used in
120 `choose` (first-choice member first).
121 """
124class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor):
125 """A construction visitor for `DatabaseTopologicalFamily`.
127 This visitor depends on (and is thus visited after) its members.
129 Parameters
130 ----------
131 name : `str`
132 Name of the family.
133 members : `~collections.abc.Iterable` [ `str` ]
134 The names of the members of this family, ordered according to the
135 priority used in `choose` (first-choice member first).
136 """
138 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]):
139 super().__init__(name)
140 self._space = space
141 self._members = tuple(members)
143 def hasDependenciesIn(self, others: Set[str]) -> bool:
144 # Docstring inherited from DimensionConstructionVisitor.
145 return not others.isdisjoint(self._members)
147 def visit(self, builder: DimensionConstructionBuilder) -> None:
148 # Docstring inherited from DimensionConstructionVisitor.
149 members = NamedValueSet(builder.elements[name] for name in self._members)
150 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze())
151 builder.topology[self._space].add(family)
152 for member in members:
153 assert isinstance(member, DatabaseDimension | DatabaseDimensionCombination)
154 other = member._topology.setdefault(self._space, family)
155 if other is not family:
156 raise RuntimeError(
157 f"{member.name} is declared to be a member of (at least) two "
158 f"{self._space.name} families: {other.name} and {family.name}."
159 )
162class DatabaseDimensionElement(DimensionElement):
163 """An intermediate base class for `DimensionElement` database classes.
165 Theese classes are ones whose instances map directly to a database
166 table or query.
168 Parameters
169 ----------
170 name : `str`
171 Name of the dimension.
172 storage : `dict`
173 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
174 that will back this element in the registry (in a "cls" key) along
175 with any other construction keyword arguments (in other keys).
176 implied : `NamedValueAbstractSet` [ `Dimension` ]
177 Other dimensions whose keys are included in this dimension's (logical)
178 table as foreign keys.
179 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
180 Field specifications for all non-key fields in this dimension's table.
181 """
183 def __init__(
184 self,
185 name: str,
186 storage: dict,
187 *,
188 implied: NamedValueAbstractSet[Dimension],
189 metadata: NamedValueAbstractSet[ddl.FieldSpec],
190 ):
191 self._name = name
192 self._storage = storage
193 self._implied = implied
194 self._metadata = metadata
195 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {}
197 @property
198 def name(self) -> str:
199 # Docstring inherited from TopologicalRelationshipEndpoint.
200 return self._name
202 @property
203 def implied(self) -> NamedValueAbstractSet[Dimension]:
204 # Docstring inherited from DimensionElement.
205 return self._implied
207 @property
208 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
209 # Docstring inherited from DimensionElement.
210 return self._metadata
212 @property
213 def viewOf(self) -> str | None:
214 # Docstring inherited from DimensionElement.
215 # This is a bit encapsulation-breaking; these storage config values
216 # are supposed to be opaque here, and just forwarded on to some
217 # DimensionRecordStorage implementation. The long-term fix is to
218 # move viewOf entirely, by changing the code that relies on it to rely
219 # on the DimensionRecordStorage object instead.
220 storage = self._storage.get("nested")
221 if storage is None:
222 storage = self._storage
223 return storage.get("view_of")
225 @property
226 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]:
227 # Docstring inherited from TopologicalRelationshipEndpoint
228 return MappingProxyType(self._topology)
230 @property
231 def spatial(self) -> DatabaseTopologicalFamily | None:
232 # Docstring inherited from TopologicalRelationshipEndpoint
233 return self.topology.get(TopologicalSpace.SPATIAL)
235 @property
236 def temporal(self) -> DatabaseTopologicalFamily | None:
237 # Docstring inherited from TopologicalRelationshipEndpoint
238 return self.topology.get(TopologicalSpace.TEMPORAL)
240 def makeStorage(
241 self,
242 db: Database,
243 *,
244 context: StaticTablesContext | None = None,
245 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
246 view_target: DatabaseDimensionRecordStorage | None = None,
247 ) -> DatabaseDimensionRecordStorage:
248 """Make the dimension record storage instance for this database.
250 Constructs the `DimensionRecordStorage` instance that should
251 be used to back this element in a registry.
253 Parameters
254 ----------
255 db : `Database`
256 Interface to the underlying database engine and namespace.
257 context : `StaticTablesContext`, optional
258 If provided, an object to use to create any new tables. If not
259 provided, ``db.ensureTableExists`` should be used instead.
260 governors : `NamedKeyMapping`
261 Mapping from `GovernorDimension` to the record storage backend for
262 that dimension, containing all governor dimensions.
263 view_target : `DatabaseDimensionRecordStorage`, optional
264 Storage object for the element this target's storage is a view of
265 (i.e. when `viewOf` is not `None`).
267 Returns
268 -------
269 storage : `DatabaseDimensionRecordStorage`
270 Storage object that should back this element in a registry.
271 """
272 from ..registry.interfaces import DatabaseDimensionRecordStorage
274 cls = doImportType(self._storage["cls"])
275 assert issubclass(cls, DatabaseDimensionRecordStorage)
276 return cls.initialize(
277 db,
278 self,
279 context=context,
280 config=self._storage,
281 governors=governors,
282 view_target=view_target,
283 )
286class DatabaseDimension(Dimension, DatabaseDimensionElement):
287 """A `Dimension` class that maps directly to a database table or query.
289 Parameters
290 ----------
291 name : `str`
292 Name of the dimension.
293 storage : `dict`
294 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
295 that will back this element in the registry (in a "cls" key) along
296 with any other construction keyword arguments (in other keys).
297 required : `NamedValueSet` [ `Dimension` ]
298 Other dimensions whose keys are part of the compound primary key for
299 this dimension's (logical) table, as well as references to their own
300 tables. The ``required`` parameter does not include ``self`` (it
301 can't, of course), but the corresponding attribute does - it is added
302 by the constructor.
303 implied : `NamedValueAbstractSet` [ `Dimension` ]
304 Other dimensions whose keys are included in this dimension's (logical)
305 table as foreign keys.
306 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
307 Field specifications for all non-key fields in this dimension's table.
308 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
309 Fields that can each be used to uniquely identify this dimension (given
310 values for all required dimensions). The first of these is used as
311 (part of) this dimension's table's primary key, while others are used
312 to define unique constraints.
314 Notes
315 -----
316 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is
317 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to
318 update the `~TopologicalRelationshipEndpoint.topology` attribute of their
319 members.
320 """
322 def __init__(
323 self,
324 name: str,
325 storage: dict,
326 *,
327 required: NamedValueSet[Dimension],
328 implied: NamedValueAbstractSet[Dimension],
329 metadata: NamedValueAbstractSet[ddl.FieldSpec],
330 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec],
331 ):
332 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
333 required.add(self)
334 self._required = required.freeze()
335 self._uniqueKeys = uniqueKeys
337 @property
338 def required(self) -> NamedValueAbstractSet[Dimension]:
339 # Docstring inherited from DimensionElement.
340 return self._required
342 @property
343 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
344 # Docstring inherited from Dimension.
345 return self._uniqueKeys
348class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement):
349 """A combination class that maps directly to a database table or query.
351 Parameters
352 ----------
353 name : `str`
354 Name of the dimension.
355 storage : `dict`
356 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
357 that will back this element in the registry (in a "cls" key) along
358 with any other construction keyword arguments (in other keys).
359 required : `NamedValueAbstractSet` [ `Dimension` ]
360 Dimensions whose keys define the compound primary key for this
361 combinations's (logical) table, as well as references to their own
362 tables.
363 implied : `NamedValueAbstractSet` [ `Dimension` ]
364 Dimensions whose keys are included in this combinations's (logical)
365 table as foreign keys.
366 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
367 Field specifications for all non-key fields in this combination's
368 table.
369 alwaysJoin : `bool`, optional
370 If `True`, always include this element in any query or data ID in
371 which its ``required`` dimensions appear, because it defines a
372 relationship between those dimensions that must always be satisfied.
374 Notes
375 -----
376 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`,
377 but it is the responsibility of
378 `DatabaseTopologicalFamilyConstructionVisitor` to update the
379 `~TopologicalRelationshipEndpoint.topology` attribute of their members.
381 This class has a lot in common with `DatabaseDimension`, but they are
382 expected to diverge in future changes, and the only way to make them share
383 method implementations would be via multiple inheritance. Given the
384 trivial nature of all of those implementations, this does not seem worth
385 the drawbacks (particularly the constraints it imposes on constructor
386 signatures).
387 """
389 def __init__(
390 self,
391 name: str,
392 storage: dict,
393 *,
394 required: NamedValueAbstractSet[Dimension],
395 implied: NamedValueAbstractSet[Dimension],
396 metadata: NamedValueAbstractSet[ddl.FieldSpec],
397 alwaysJoin: bool,
398 populated_by: Dimension | None,
399 ):
400 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
401 self._required = required
402 self._alwaysJoin = alwaysJoin
403 self._populated_by = populated_by
405 @property
406 def required(self) -> NamedValueAbstractSet[Dimension]:
407 # Docstring inherited from DimensionElement.
408 return self._required
410 @property
411 def alwaysJoin(self) -> bool:
412 # Docstring inherited from DimensionElement.
413 return self._alwaysJoin
415 @property
416 def populated_by(self) -> Dimension | None:
417 # Docstring inherited.
418 return self._populated_by
421class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor):
422 """Construction visitor for database dimension and dimension combination.
424 Specifically, a construction visitor for `DatabaseDimension` and
425 `DatabaseDimensionCombination`.
427 Parameters
428 ----------
429 name : `str`
430 Name of the dimension.
431 storage : `dict`
432 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
433 that will back this element in the registry (in a "cls" key) along
434 with any other construction keyword arguments (in other keys).
435 required : `~collections.abc.Set` [ `Dimension` ]
436 Names of dimensions whose keys define the compound primary key for this
437 element's (logical) table, as well as references to their own
438 tables.
439 implied : `~collections.abc.Set` [ `Dimension` ]
440 Names of dimension whose keys are included in this elements's
441 (logical) table as foreign keys.
442 metadata : `~collections.abc.Iterable` [ `ddl.FieldSpec` ]
443 Field specifications for all non-key fields in this element's table.
444 uniqueKeys : `~collections.abc.Iterable` [ `ddl.FieldSpec` ]
445 Fields that can each be used to uniquely identify this dimension (given
446 values for all required dimensions). The first of these is used as
447 (part of) this dimension's table's primary key, while others are used
448 to define unique constraints. Should be empty for
449 `DatabaseDimensionCombination` definitions.
450 alwaysJoin : `bool`, optional
451 If `True`, always include this element in any query or data ID in
452 which its ``required`` dimensions appear, because it defines a
453 relationship between those dimensions that must always be satisfied.
454 Should only be provided when a `DimensionCombination` is being
455 constructed.
456 populated_by: `Dimension`, optional
457 The dimension that this element's records are always inserted,
458 exported, and imported alongside.
459 """
461 def __init__(
462 self,
463 name: str,
464 storage: dict,
465 required: set[str],
466 implied: set[str],
467 metadata: Iterable[ddl.FieldSpec] = (),
468 uniqueKeys: Iterable[ddl.FieldSpec] = (),
469 alwaysJoin: bool = False,
470 populated_by: str | None = None,
471 ):
472 super().__init__(name)
473 self._storage = storage
474 self._required = required
475 self._implied = implied
476 self._metadata = NamedValueSet(metadata).freeze()
477 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze()
478 self._alwaysJoin = alwaysJoin
479 self._populated_by = populated_by
481 def hasDependenciesIn(self, others: Set[str]) -> bool:
482 # Docstring inherited from DimensionConstructionVisitor.
483 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others))
485 def visit(self, builder: DimensionConstructionBuilder) -> None:
486 # Docstring inherited from DimensionConstructionVisitor.
487 # Expand required dependencies.
488 for name in tuple(self._required): # iterate over copy
489 self._required.update(builder.dimensions[name].required.names)
490 # Transform required and implied Dimension names into instances,
491 # and reorder to match builder's order.
492 required: NamedValueSet[Dimension] = NamedValueSet()
493 implied: NamedValueSet[Dimension] = NamedValueSet()
494 for dimension in builder.dimensions:
495 if dimension.name in self._required:
496 required.add(dimension)
497 if dimension.name in self._implied:
498 implied.add(dimension)
500 if self._uniqueKeys:
501 if self._alwaysJoin:
502 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.")
503 # Special handling for creating Dimension instances.
504 dimension = DatabaseDimension(
505 self.name,
506 storage=self._storage,
507 required=required,
508 implied=implied.freeze(),
509 metadata=self._metadata,
510 uniqueKeys=self._uniqueKeys,
511 )
512 builder.dimensions.add(dimension)
513 builder.elements.add(dimension)
514 else:
515 # Special handling for creating DimensionCombination instances.
516 combination = DatabaseDimensionCombination(
517 self.name,
518 storage=self._storage,
519 required=required,
520 implied=implied.freeze(),
521 metadata=self._metadata,
522 alwaysJoin=self._alwaysJoin,
523 populated_by=(
524 builder.dimensions[self._populated_by] if self._populated_by is not None else None
525 ),
526 )
527 builder.elements.add(combination)