Coverage for python/lsst/daf/butler/dimensions/_database.py: 44%
143 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = (
31 "DatabaseDimension",
32 "DatabaseDimensionCombination",
33 "DatabaseDimensionElement",
34 "DatabaseTopologicalFamily",
35)
37from collections.abc import Iterable, Mapping, Set
38from types import MappingProxyType
39from typing import TYPE_CHECKING
41from lsst.utils import doImportType
42from lsst.utils.classes import cached_getter
44from .. import ddl
45from .._named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet
46from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace
47from ._elements import Dimension, DimensionCombination, DimensionElement
48from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
50if TYPE_CHECKING:
51 from ..registry.interfaces import (
52 Database,
53 DatabaseDimensionRecordStorage,
54 GovernorDimensionRecordStorage,
55 StaticTablesContext,
56 )
57 from ._governor import GovernorDimension
60class DatabaseTopologicalFamily(TopologicalFamily):
61 """Database topological family implementation.
63 A `TopologicalFamily` implementation for the `DatabaseDimension` and
64 `DatabaseDimensionCombination` objects that have direct database
65 representations.
67 Parameters
68 ----------
69 name : `str`
70 Name of the family.
71 space : `TopologicalSpace`
72 Space in which this family's regions live.
73 members : `NamedValueAbstractSet` [ `DimensionElement` ]
74 The members of this family, ordered according to the priority used
75 in `choose` (first-choice member first).
76 """
78 def __init__(
79 self,
80 name: str,
81 space: TopologicalSpace,
82 *,
83 members: NamedValueAbstractSet[DimensionElement],
84 ):
85 super().__init__(name, space)
86 self.members = members
88 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement:
89 # Docstring inherited from TopologicalFamily.
90 for member in self.members:
91 if member in endpoints:
92 return member
93 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.")
95 @property
96 @cached_getter
97 def governor(self) -> GovernorDimension:
98 """Return `GovernorDimension` common to all members of this family.
100 (`GovernorDimension`).
101 """
102 governors = {m.governor for m in self.members}
103 if None in governors:
104 raise RuntimeError(
105 f"Bad {self.space.name} family definition {self.name}: at least one member "
106 f"in {self.members} has no GovernorDimension dependency."
107 )
108 try:
109 (result,) = governors
110 except ValueError:
111 raise RuntimeError(
112 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} "
113 f"in {self.members}."
114 ) from None
115 return result # type: ignore
117 members: NamedValueAbstractSet[DimensionElement]
118 """The members of this family, ordered according to the priority used in
119 `choose` (first-choice member first).
120 """
123class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor):
124 """A construction visitor for `DatabaseTopologicalFamily`.
126 This visitor depends on (and is thus visited after) its members.
128 Parameters
129 ----------
130 name : `str`
131 Name of the family.
132 members : `~collections.abc.Iterable` [ `str` ]
133 The names of the members of this family, ordered according to the
134 priority used in `choose` (first-choice member first).
135 """
137 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]):
138 super().__init__(name)
139 self._space = space
140 self._members = tuple(members)
142 def hasDependenciesIn(self, others: Set[str]) -> bool:
143 # Docstring inherited from DimensionConstructionVisitor.
144 return not others.isdisjoint(self._members)
146 def visit(self, builder: DimensionConstructionBuilder) -> None:
147 # Docstring inherited from DimensionConstructionVisitor.
148 members = NamedValueSet(builder.elements[name] for name in self._members)
149 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze())
150 builder.topology[self._space].add(family)
151 for member in members:
152 assert isinstance(member, DatabaseDimension | DatabaseDimensionCombination)
153 other = member._topology.setdefault(self._space, family)
154 if other is not family:
155 raise RuntimeError(
156 f"{member.name} is declared to be a member of (at least) two "
157 f"{self._space.name} families: {other.name} and {family.name}."
158 )
161class DatabaseDimensionElement(DimensionElement):
162 """An intermediate base class for `DimensionElement` database classes.
164 Theese classes are ones whose instances map directly to a database
165 table or query.
167 Parameters
168 ----------
169 name : `str`
170 Name of the dimension.
171 storage : `dict`
172 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
173 that will back this element in the registry (in a "cls" key) along
174 with any other construction keyword arguments (in other keys).
175 implied : `NamedValueAbstractSet` [ `Dimension` ]
176 Other dimensions whose keys are included in this dimension's (logical)
177 table as foreign keys.
178 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
179 Field specifications for all non-key fields in this dimension's table.
180 """
182 def __init__(
183 self,
184 name: str,
185 storage: dict,
186 *,
187 implied: NamedValueAbstractSet[Dimension],
188 metadata: NamedValueAbstractSet[ddl.FieldSpec],
189 ):
190 self._name = name
191 self._storage = storage
192 self._implied = implied
193 self._metadata = metadata
194 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {}
196 @property
197 def name(self) -> str:
198 # Docstring inherited from TopologicalRelationshipEndpoint.
199 return self._name
201 @property
202 def implied(self) -> NamedValueAbstractSet[Dimension]:
203 # Docstring inherited from DimensionElement.
204 return self._implied
206 @property
207 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
208 # Docstring inherited from DimensionElement.
209 return self._metadata
211 @property
212 def viewOf(self) -> str | None:
213 # Docstring inherited from DimensionElement.
214 # This is a bit encapsulation-breaking; these storage config values
215 # are supposed to be opaque here, and just forwarded on to some
216 # DimensionRecordStorage implementation. The long-term fix is to
217 # move viewOf entirely, by changing the code that relies on it to rely
218 # on the DimensionRecordStorage object instead.
219 storage = self._storage.get("nested")
220 if storage is None:
221 storage = self._storage
222 return storage.get("view_of")
224 @property
225 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]:
226 # Docstring inherited from TopologicalRelationshipEndpoint
227 return MappingProxyType(self._topology)
229 @property
230 def spatial(self) -> DatabaseTopologicalFamily | None:
231 # Docstring inherited from TopologicalRelationshipEndpoint
232 return self.topology.get(TopologicalSpace.SPATIAL)
234 @property
235 def temporal(self) -> DatabaseTopologicalFamily | None:
236 # Docstring inherited from TopologicalRelationshipEndpoint
237 return self.topology.get(TopologicalSpace.TEMPORAL)
239 def makeStorage(
240 self,
241 db: Database,
242 *,
243 context: StaticTablesContext | None = None,
244 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
245 view_target: DatabaseDimensionRecordStorage | None = None,
246 ) -> DatabaseDimensionRecordStorage:
247 """Make the dimension record storage instance for this database.
249 Constructs the `DimensionRecordStorage` instance that should
250 be used to back this element in a registry.
252 Parameters
253 ----------
254 db : `Database`
255 Interface to the underlying database engine and namespace.
256 context : `StaticTablesContext`, optional
257 If provided, an object to use to create any new tables. If not
258 provided, ``db.ensureTableExists`` should be used instead.
259 governors : `NamedKeyMapping`
260 Mapping from `GovernorDimension` to the record storage backend for
261 that dimension, containing all governor dimensions.
262 view_target : `DatabaseDimensionRecordStorage`, optional
263 Storage object for the element this target's storage is a view of
264 (i.e. when `viewOf` is not `None`).
266 Returns
267 -------
268 storage : `DatabaseDimensionRecordStorage`
269 Storage object that should back this element in a registry.
270 """
271 from ..registry.interfaces import DatabaseDimensionRecordStorage
273 cls = doImportType(self._storage["cls"])
274 assert issubclass(cls, DatabaseDimensionRecordStorage)
275 return cls.initialize(
276 db,
277 self,
278 context=context,
279 config=self._storage,
280 governors=governors,
281 view_target=view_target,
282 )
285class DatabaseDimension(Dimension, DatabaseDimensionElement):
286 """A `Dimension` class that maps directly to a database table or query.
288 Parameters
289 ----------
290 name : `str`
291 Name of the dimension.
292 storage : `dict`
293 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
294 that will back this element in the registry (in a "cls" key) along
295 with any other construction keyword arguments (in other keys).
296 required : `NamedValueSet` [ `Dimension` ]
297 Other dimensions whose keys are part of the compound primary key for
298 this dimension's (logical) table, as well as references to their own
299 tables. The ``required`` parameter does not include ``self`` (it
300 can't, of course), but the corresponding attribute does - it is added
301 by the constructor.
302 implied : `NamedValueAbstractSet` [ `Dimension` ]
303 Other dimensions whose keys are included in this dimension's (logical)
304 table as foreign keys.
305 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
306 Field specifications for all non-key fields in this dimension's table.
307 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
308 Fields that can each be used to uniquely identify this dimension (given
309 values for all required dimensions). The first of these is used as
310 (part of) this dimension's table's primary key, while others are used
311 to define unique constraints.
313 Notes
314 -----
315 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is
316 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to
317 update the `~TopologicalRelationshipEndpoint.topology` attribute of their
318 members.
319 """
321 def __init__(
322 self,
323 name: str,
324 storage: dict,
325 *,
326 required: NamedValueSet[Dimension],
327 implied: NamedValueAbstractSet[Dimension],
328 metadata: NamedValueAbstractSet[ddl.FieldSpec],
329 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec],
330 ):
331 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
332 required.add(self)
333 self._required = required.freeze()
334 self._uniqueKeys = uniqueKeys
336 @property
337 def required(self) -> NamedValueAbstractSet[Dimension]:
338 # Docstring inherited from DimensionElement.
339 return self._required
341 @property
342 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
343 # Docstring inherited from Dimension.
344 return self._uniqueKeys
347class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement):
348 """A combination class that maps directly to a database table or query.
350 Parameters
351 ----------
352 name : `str`
353 Name of the dimension.
354 storage : `dict`
355 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
356 that will back this element in the registry (in a "cls" key) along
357 with any other construction keyword arguments (in other keys).
358 required : `NamedValueAbstractSet` [ `Dimension` ]
359 Dimensions whose keys define the compound primary key for this
360 combinations's (logical) table, as well as references to their own
361 tables.
362 implied : `NamedValueAbstractSet` [ `Dimension` ]
363 Dimensions whose keys are included in this combinations's (logical)
364 table as foreign keys.
365 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ]
366 Field specifications for all non-key fields in this combination's
367 table.
368 alwaysJoin : `bool`, optional
369 If `True`, always include this element in any query or data ID in
370 which its ``required`` dimensions appear, because it defines a
371 relationship between those dimensions that must always be satisfied.
373 Notes
374 -----
375 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`,
376 but it is the responsibility of
377 `DatabaseTopologicalFamilyConstructionVisitor` to update the
378 `~TopologicalRelationshipEndpoint.topology` attribute of their members.
380 This class has a lot in common with `DatabaseDimension`, but they are
381 expected to diverge in future changes, and the only way to make them share
382 method implementations would be via multiple inheritance. Given the
383 trivial nature of all of those implementations, this does not seem worth
384 the drawbacks (particularly the constraints it imposes on constructor
385 signatures).
386 """
388 def __init__(
389 self,
390 name: str,
391 storage: dict,
392 *,
393 required: NamedValueAbstractSet[Dimension],
394 implied: NamedValueAbstractSet[Dimension],
395 metadata: NamedValueAbstractSet[ddl.FieldSpec],
396 alwaysJoin: bool,
397 populated_by: Dimension | None,
398 ):
399 super().__init__(name, storage=storage, implied=implied, metadata=metadata)
400 self._required = required
401 self._alwaysJoin = alwaysJoin
402 self._populated_by = populated_by
404 @property
405 def required(self) -> NamedValueAbstractSet[Dimension]:
406 # Docstring inherited from DimensionElement.
407 return self._required
409 @property
410 def alwaysJoin(self) -> bool:
411 # Docstring inherited from DimensionElement.
412 return self._alwaysJoin
414 @property
415 def populated_by(self) -> Dimension | None:
416 # Docstring inherited.
417 return self._populated_by
420class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor):
421 """Construction visitor for database dimension and dimension combination.
423 Specifically, a construction visitor for `DatabaseDimension` and
424 `DatabaseDimensionCombination`.
426 Parameters
427 ----------
428 name : `str`
429 Name of the dimension.
430 storage : `dict`
431 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass
432 that will back this element in the registry (in a "cls" key) along
433 with any other construction keyword arguments (in other keys).
434 required : `~collections.abc.Set` [ `Dimension` ]
435 Names of dimensions whose keys define the compound primary key for this
436 element's (logical) table, as well as references to their own
437 tables.
438 implied : `~collections.abc.Set` [ `Dimension` ]
439 Names of dimension whose keys are included in this elements's
440 (logical) table as foreign keys.
441 metadata : `~collections.abc.Iterable` [ `ddl.FieldSpec` ]
442 Field specifications for all non-key fields in this element's table.
443 uniqueKeys : `~collections.abc.Iterable` [ `ddl.FieldSpec` ]
444 Fields that can each be used to uniquely identify this dimension (given
445 values for all required dimensions). The first of these is used as
446 (part of) this dimension's table's primary key, while others are used
447 to define unique constraints. Should be empty for
448 `DatabaseDimensionCombination` definitions.
449 alwaysJoin : `bool`, optional
450 If `True`, always include this element in any query or data ID in
451 which its ``required`` dimensions appear, because it defines a
452 relationship between those dimensions that must always be satisfied.
453 Should only be provided when a `DimensionCombination` is being
454 constructed.
455 populated_by: `Dimension`, optional
456 The dimension that this element's records are always inserted,
457 exported, and imported alongside.
458 """
460 def __init__(
461 self,
462 name: str,
463 storage: dict,
464 required: set[str],
465 implied: set[str],
466 metadata: Iterable[ddl.FieldSpec] = (),
467 uniqueKeys: Iterable[ddl.FieldSpec] = (),
468 alwaysJoin: bool = False,
469 populated_by: str | None = None,
470 ):
471 super().__init__(name)
472 self._storage = storage
473 self._required = required
474 self._implied = implied
475 self._metadata = NamedValueSet(metadata).freeze()
476 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze()
477 self._alwaysJoin = alwaysJoin
478 self._populated_by = populated_by
480 def hasDependenciesIn(self, others: Set[str]) -> bool:
481 # Docstring inherited from DimensionConstructionVisitor.
482 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others))
484 def visit(self, builder: DimensionConstructionBuilder) -> None:
485 # Docstring inherited from DimensionConstructionVisitor.
486 # Expand required dependencies.
487 for name in tuple(self._required): # iterate over copy
488 self._required.update(builder.dimensions[name].required.names)
489 # Transform required and implied Dimension names into instances,
490 # and reorder to match builder's order.
491 required: NamedValueSet[Dimension] = NamedValueSet()
492 implied: NamedValueSet[Dimension] = NamedValueSet()
493 for dimension in builder.dimensions:
494 if dimension.name in self._required:
495 required.add(dimension)
496 if dimension.name in self._implied:
497 implied.add(dimension)
499 if self._uniqueKeys:
500 if self._alwaysJoin:
501 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.")
502 # Special handling for creating Dimension instances.
503 dimension = DatabaseDimension(
504 self.name,
505 storage=self._storage,
506 required=required,
507 implied=implied.freeze(),
508 metadata=self._metadata,
509 uniqueKeys=self._uniqueKeys,
510 )
511 builder.dimensions.add(dimension)
512 builder.elements.add(dimension)
513 else:
514 # Special handling for creating DimensionCombination instances.
515 combination = DatabaseDimensionCombination(
516 self.name,
517 storage=self._storage,
518 required=required,
519 implied=implied.freeze(),
520 metadata=self._metadata,
521 alwaysJoin=self._alwaysJoin,
522 populated_by=(
523 builder.dimensions[self._populated_by] if self._populated_by is not None else None
524 ),
525 )
526 builder.elements.add(combination)