Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 69%
118 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "DatabaseDimensionOverlapStorage",
25 "DatabaseDimensionRecordStorage",
26 "DimensionRecordStorage",
27 "DimensionRecordStorageManager",
28 "GovernorDimensionRecordStorage",
29 "SkyPixDimensionRecordStorage",
30)
32from abc import ABC, abstractmethod
33from typing import (
34 AbstractSet, Any,
35 Callable,
36 Dict,
37 Iterable, Mapping,
38 Optional,
39 Tuple,
40 TYPE_CHECKING,
41 Union,
42)
44import sqlalchemy
46from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension
47from ._versioning import VersionedExtension
49if TYPE_CHECKING: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true
50 from ...core import (
51 DataCoordinateIterable,
52 DimensionElement,
53 DimensionRecord,
54 DimensionUniverse,
55 NamedKeyDict,
56 NamedKeyMapping,
57 TimespanDatabaseRepresentation,
58 )
59 from ..queries import QueryBuilder
60 from ._database import Database, StaticTablesContext
63OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]]
66class DimensionRecordStorage(ABC):
67 """An abstract base class that represents a way of storing the records
68 associated with a single `DimensionElement`.
70 Concrete `DimensionRecordStorage` instances should generally be constructed
71 via a call to `setupDimensionStorage`, which selects the appropriate
72 subclass for each element according to its configuration.
74 All `DimensionRecordStorage` methods are pure abstract, even though in some
75 cases a reasonable default implementation might be possible, in order to
76 better guarantee all methods are correctly overridden. All of these
77 potentially-defaultable implementations are extremely trivial, so asking
78 subclasses to provide them is not a significant burden.
79 """
81 @property
82 @abstractmethod
83 def element(self) -> DimensionElement:
84 """The element whose records this instance holds (`DimensionElement`).
85 """
86 raise NotImplementedError()
88 @abstractmethod
89 def clearCaches(self) -> None:
90 """Clear any in-memory caches held by the storage instance.
92 This is called by `Registry` when transactions are rolled back, to
93 avoid in-memory caches from ever containing records that are not
94 present in persistent storage.
95 """
96 raise NotImplementedError()
98 @abstractmethod
99 def join(
100 self,
101 builder: QueryBuilder, *,
102 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None,
103 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None,
104 ) -> sqlalchemy.sql.FromClause:
105 """Add the dimension element's logical table to a query under
106 construction.
108 This is a visitor pattern interface that is expected to be called only
109 by `QueryBuilder.joinDimensionElement`.
111 Parameters
112 ----------
113 builder : `QueryBuilder`
114 Builder for the query that should contain this element.
115 regions : `NamedKeyDict`, optional
116 A mapping from `DimensionElement` to a SQLAlchemy column containing
117 the region for that element, which should be updated to include a
118 region column for this element if one exists. If `None`,
119 ``self.element`` is not being included in the query via a spatial
120 join.
121 timespan : `NamedKeyDict`, optional
122 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
123 columns containing the timespan for that element, which should be
124 updated to include timespan columns for this element if they exist.
125 If `None`, ``self.element`` is not being included in the query via
126 a temporal join.
128 Returns
129 -------
130 fromClause : `sqlalchemy.sql.FromClause`
131 Table or clause for the element which is joined.
133 Notes
134 -----
135 Elements are only included in queries via spatial and/or temporal joins
136 when necessary to connect them to other elements in the query, so
137 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
138 because an element has a region or timespan.
139 """
140 raise NotImplementedError()
142 @abstractmethod
143 def insert(self, *records: DimensionRecord, replace: bool = False) -> None:
144 """Insert one or more records into storage.
146 Parameters
147 ----------
148 records
149 One or more instances of the `DimensionRecord` subclass for the
150 element this storage is associated with.
151 replace: `bool`, optional
152 If `True` (`False` is default), replace existing records in the
153 database if there is a conflict.
155 Raises
156 ------
157 TypeError
158 Raised if the element does not support record insertion.
159 sqlalchemy.exc.IntegrityError
160 Raised if one or more records violate database integrity
161 constraints.
163 Notes
164 -----
165 As `insert` is expected to be called only by a `Registry`, we rely
166 on `Registry` to provide transactionality, both by using a SQLALchemy
167 connection shared with the `Registry` and by relying on it to call
168 `clearCaches` when rolling back transactions.
169 """
170 raise NotImplementedError()
172 @abstractmethod
173 def sync(self, record: DimensionRecord, update: bool = False) -> Union[bool, Dict[str, Any]]:
174 """Synchronize a record with the database, inserting it only if it does
175 not exist and comparing values if it does.
177 Parameters
178 ----------
179 record : `DimensionRecord`.
180 An instance of the `DimensionRecord` subclass for the
181 element this storage is associated with.
182 update: `bool`, optional
183 If `True` (`False` is default), update the existing record in the
184 database if there is a conflict.
186 Returns
187 -------
188 inserted_or_updated : `bool` or `dict`
189 `True` if a new row was inserted, `False` if no changes were
190 needed, or a `dict` mapping updated column names to their old
191 values if an update was performed (only possible if
192 ``update=True``).
194 Raises
195 ------
196 DatabaseConflictError
197 Raised if the record exists in the database (according to primary
198 key lookup) but is inconsistent with the given one.
199 TypeError
200 Raised if the element does not support record synchronization.
201 sqlalchemy.exc.IntegrityError
202 Raised if one or more records violate database integrity
203 constraints.
204 """
205 raise NotImplementedError()
207 @abstractmethod
208 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
209 """Retrieve records from storage.
211 Parameters
212 ----------
213 dataIds : `DataCoordinateIterable`
214 Data IDs that identify the records to be retrieved.
216 Returns
217 -------
218 records : `Iterable` [ `DimensionRecord` ]
219 Record retrieved from storage. Not all data IDs may have
220 corresponding records (if there are no records that match a data
221 ID), and even if they are, the order of inputs is not preserved.
222 """
223 raise NotImplementedError()
225 @abstractmethod
226 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
227 """Return tables used for schema digest.
229 Returns
230 -------
231 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
232 Possibly empty set of tables for schema digest calculations.
233 """
234 raise NotImplementedError()
237class GovernorDimensionRecordStorage(DimensionRecordStorage):
238 """Intermediate interface for `DimensionRecordStorage` objects that provide
239 storage for `GovernorDimension` instances.
240 """
242 @classmethod
243 @abstractmethod
244 def initialize(cls, db: Database, dimension: GovernorDimension, *,
245 context: Optional[StaticTablesContext] = None,
246 config: Mapping[str, Any],
247 ) -> GovernorDimensionRecordStorage:
248 """Construct an instance of this class using a standardized interface.
250 Parameters
251 ----------
252 db : `Database`
253 Interface to the underlying database engine and namespace.
254 dimension : `GovernorDimension`
255 Dimension the new instance will manage records for.
256 context : `StaticTablesContext`, optional
257 If provided, an object to use to create any new tables. If not
258 provided, ``db.ensureTableExists`` should be used instead.
259 config : `Mapping`
260 Extra configuration options specific to the implementation.
262 Returns
263 -------
264 storage : `GovernorDimensionRecordStorage`
265 A new `GovernorDimensionRecordStorage` subclass instance.
266 """
267 raise NotImplementedError()
269 @property
270 @abstractmethod
271 def element(self) -> GovernorDimension:
272 # Docstring inherited from DimensionRecordStorage.
273 raise NotImplementedError()
275 @abstractmethod
276 def refresh(self) -> None:
277 """Ensure all other operations on this manager are aware of any
278 changes made by other clients since it was initialized or last
279 refreshed.
280 """
281 raise NotImplementedError()
283 @property
284 @abstractmethod
285 def values(self) -> AbstractSet[str]:
286 """All primary key values for this dimension (`set` [ `str` ]).
288 This may rely on an in-memory cache and hence not reflect changes to
289 the set of values made by other `Butler` / `Registry` clients. Call
290 `refresh` to ensure up-to-date results.
291 """
292 raise NotImplementedError()
294 @property
295 @abstractmethod
296 def table(self) -> sqlalchemy.schema.Table:
297 """The SQLAlchemy table that backs this dimension
298 (`sqlalchemy.schema.Table`).
299 """
300 raise NotImplementedError
302 @abstractmethod
303 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None:
304 """Add a function or method to be called after new records for this
305 dimension are inserted by `insert` or `sync`.
307 Parameters
308 ----------
309 callback
310 Callable that takes a single `DimensionRecord` argument. This will
311 be called immediately after any successful insertion, in the same
312 transaction.
313 """
314 raise NotImplementedError()
317class SkyPixDimensionRecordStorage(DimensionRecordStorage):
318 """Intermediate interface for `DimensionRecordStorage` objects that provide
319 storage for `SkyPixDimension` instances.
320 """
322 @property
323 @abstractmethod
324 def element(self) -> SkyPixDimension:
325 # Docstring inherited from DimensionRecordStorage.
326 raise NotImplementedError()
329class DatabaseDimensionRecordStorage(DimensionRecordStorage):
330 """Intermediate interface for `DimensionRecordStorage` objects that provide
331 storage for `DatabaseDimensionElement` instances.
332 """
334 @classmethod
335 @abstractmethod
336 def initialize(cls, db: Database, element: DatabaseDimensionElement, *,
337 context: Optional[StaticTablesContext] = None,
338 config: Mapping[str, Any],
339 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
340 ) -> DatabaseDimensionRecordStorage:
341 """Construct an instance of this class using a standardized interface.
343 Parameters
344 ----------
345 db : `Database`
346 Interface to the underlying database engine and namespace.
347 element : `DatabaseDimensionElement`
348 Dimension element the new instance will manage records for.
349 context : `StaticTablesContext`, optional
350 If provided, an object to use to create any new tables. If not
351 provided, ``db.ensureTableExists`` should be used instead.
352 config : `Mapping`
353 Extra configuration options specific to the implementation.
354 governors : `NamedKeyMapping`
355 Mapping containing all governor dimension storage implementations.
357 Returns
358 -------
359 storage : `DatabaseDimensionRecordStorage`
360 A new `DatabaseDimensionRecordStorage` subclass instance.
361 """
362 raise NotImplementedError()
364 @property
365 @abstractmethod
366 def element(self) -> DatabaseDimensionElement:
367 # Docstring inherited from DimensionRecordStorage.
368 raise NotImplementedError()
370 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None:
371 """Inform this record storage object of the object that will manage
372 the overlaps between this element and another element.
374 This will only be called if ``self.element.spatial is not None``,
375 and will be called immediately after construction (before any other
376 methods). In the future, implementations will be required to call a
377 method on any connected overlap storage objects any time new records
378 for the element are inserted.
380 Parameters
381 ----------
382 overlaps : `DatabaseDimensionRecordStorage`
383 Object managing overlaps between this element and another
384 database-backed element.
385 """
386 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.")
389class DatabaseDimensionOverlapStorage(ABC):
390 """A base class for objects that manage overlaps between a pair of
391 database-backed dimensions.
392 """
394 @classmethod
395 @abstractmethod
396 def initialize(
397 cls,
398 db: Database,
399 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage],
400 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage],
401 context: Optional[StaticTablesContext] = None,
402 ) -> DatabaseDimensionOverlapStorage:
403 """Construct an instance of this class using a standardized interface.
405 Parameters
406 ----------
407 db : `Database`
408 Interface to the underlying database engine and namespace.
409 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ]
410 Storage objects for the elements this object will related.
411 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ]
412 Storage objects for the governor dimensions of the elements this
413 object will related.
414 context : `StaticTablesContext`, optional
415 If provided, an object to use to create any new tables. If not
416 provided, ``db.ensureTableExists`` should be used instead.
418 Returns
419 -------
420 storage : `DatabaseDimensionOverlapStorage`
421 A new `DatabaseDimensionOverlapStorage` subclass instance.
422 """
423 raise NotImplementedError()
425 @property
426 @abstractmethod
427 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]:
428 """The pair of elements whose overlaps this object manages.
430 The order of elements is the same as their ordering within the
431 `DimensionUniverse`.
432 """
433 raise NotImplementedError()
435 @abstractmethod
436 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
437 """Return tables used for schema digest.
439 Returns
440 -------
441 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
442 Possibly empty set of tables for schema digest calculations.
443 """
444 raise NotImplementedError()
447class DimensionRecordStorageManager(VersionedExtension):
448 """An interface for managing the dimension records in a `Registry`.
450 `DimensionRecordStorageManager` primarily serves as a container and factory
451 for `DimensionRecordStorage` instances, which each provide access to the
452 records for a different `DimensionElement`.
454 Parameters
455 ----------
456 universe : `DimensionUniverse`
457 Universe of all dimensions and dimension elements known to the
458 `Registry`.
460 Notes
461 -----
462 In a multi-layer `Registry`, many dimension elements will only have
463 records in one layer (often the base layer). The union of the records
464 across all layers forms the logical table for the full `Registry`.
465 """
466 def __init__(self, *, universe: DimensionUniverse):
467 self.universe = universe
469 @classmethod
470 @abstractmethod
471 def initialize(cls, db: Database, context: StaticTablesContext, *,
472 universe: DimensionUniverse) -> DimensionRecordStorageManager:
473 """Construct an instance of the manager.
475 Parameters
476 ----------
477 db : `Database`
478 Interface to the underlying database engine and namespace.
479 context : `StaticTablesContext`
480 Context object obtained from `Database.declareStaticTables`; used
481 to declare any tables that should always be present in a layer
482 implemented with this manager.
483 universe : `DimensionUniverse`
484 Universe graph containing dimensions known to this `Registry`.
486 Returns
487 -------
488 manager : `DimensionRecordStorageManager`
489 An instance of a concrete `DimensionRecordStorageManager` subclass.
490 """
491 raise NotImplementedError()
493 @abstractmethod
494 def refresh(self) -> None:
495 """Ensure all other operations on this manager are aware of any
496 changes made by other clients since it was initialized or last
497 refreshed.
498 """
499 raise NotImplementedError()
501 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
502 """Interface to `get` that raises `LookupError` instead of returning
503 `None` on failure.
504 """
505 r = self.get(element)
506 if r is None:
507 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
508 return r
510 @abstractmethod
511 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]:
512 """Return an object that provides access to the records associated with
513 the given element, if one exists in this layer.
515 Parameters
516 ----------
517 element : `DimensionElement`
518 Element for which records should be returned.
520 Returns
521 -------
522 records : `DimensionRecordStorage` or `None`
523 The object representing the records for the given element in this
524 layer, or `None` if there are no records for that element in this
525 layer.
527 Notes
528 -----
529 Dimension elements registered by another client of the same layer since
530 the last call to `initialize` or `refresh` may not be found.
531 """
532 raise NotImplementedError()
534 @abstractmethod
535 def register(self, element: DimensionElement) -> DimensionRecordStorage:
536 """Ensure that this layer can hold records for the given element,
537 creating new tables as necessary.
539 Parameters
540 ----------
541 element : `DimensionElement`
542 Element for which a table should created (as necessary) and
543 an associated `DimensionRecordStorage` returned.
545 Returns
546 -------
547 records : `DimensionRecordStorage`
548 The object representing the records for the given element in this
549 layer.
551 Raises
552 ------
553 TransactionInterruption
554 Raised if this operation is invoked within a `Database.transaction`
555 context.
556 """
557 raise NotImplementedError()
559 @abstractmethod
560 def saveDimensionGraph(self, graph: DimensionGraph) -> int:
561 """Save a `DimensionGraph` definition to the database, allowing it to
562 be retrieved later via the returned key.
564 Parameters
565 ----------
566 graph : `DimensionGraph`
567 Set of dimensions to save.
569 Returns
570 -------
571 key : `int`
572 Integer used as the unique key for this `DimensionGraph` in the
573 database.
575 Raises
576 ------
577 TransactionInterruption
578 Raised if this operation is invoked within a `Database.transaction`
579 context.
580 """
581 raise NotImplementedError()
583 @abstractmethod
584 def loadDimensionGraph(self, key: int) -> DimensionGraph:
585 """Retrieve a `DimensionGraph` that was previously saved in the
586 database.
588 Parameters
589 ----------
590 key : `int`
591 Integer used as the unique key for this `DimensionGraph` in the
592 database.
594 Returns
595 -------
596 graph : `DimensionGraph`
597 Retrieved graph.
599 Raises
600 ------
601 KeyError
602 Raised if the given key cannot be found in the database.
603 """
604 raise NotImplementedError()
606 @abstractmethod
607 def clearCaches(self) -> None:
608 """Clear any in-memory caches held by nested `DimensionRecordStorage`
609 instances.
611 This is called by `Registry` when transactions are rolled back, to
612 avoid in-memory caches from ever containing records that are not
613 present in persistent storage.
614 """
615 raise NotImplementedError()
617 universe: DimensionUniverse
618 """Universe of all dimensions and dimension elements known to the
619 `Registry` (`DimensionUniverse`).
620 """