Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 69%
119 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:01 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:01 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "DatabaseDimensionOverlapStorage",
25 "DatabaseDimensionRecordStorage",
26 "DimensionRecordStorage",
27 "DimensionRecordStorageManager",
28 "GovernorDimensionRecordStorage",
29 "SkyPixDimensionRecordStorage",
30)
32from abc import ABC, abstractmethod
33from collections.abc import Callable, Iterable, Mapping, Set
34from typing import TYPE_CHECKING, Any, Tuple, Union
36import sqlalchemy
38from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension
39from ._versioning import VersionedExtension
41if TYPE_CHECKING: 41 ↛ 42line 41 didn't jump to line 42, because the condition on line 41 was never true
42 from ...core import (
43 DataCoordinateIterable,
44 DimensionElement,
45 DimensionRecord,
46 DimensionUniverse,
47 NamedKeyDict,
48 NamedKeyMapping,
49 TimespanDatabaseRepresentation,
50 )
51 from ..queries import QueryBuilder
52 from ._database import Database, StaticTablesContext
55OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]]
58class DimensionRecordStorage(ABC):
59 """An abstract base class that represents a way of storing the records
60 associated with a single `DimensionElement`.
62 Concrete `DimensionRecordStorage` instances should generally be constructed
63 via a call to `setupDimensionStorage`, which selects the appropriate
64 subclass for each element according to its configuration.
66 All `DimensionRecordStorage` methods are pure abstract, even though in some
67 cases a reasonable default implementation might be possible, in order to
68 better guarantee all methods are correctly overridden. All of these
69 potentially-defaultable implementations are extremely trivial, so asking
70 subclasses to provide them is not a significant burden.
71 """
73 @property
74 @abstractmethod
75 def element(self) -> DimensionElement:
76 """The element whose records this instance managers
77 (`DimensionElement`).
78 """
79 raise NotImplementedError()
81 @abstractmethod
82 def clearCaches(self) -> None:
83 """Clear any in-memory caches held by the storage instance.
85 This is called by `Registry` when transactions are rolled back, to
86 avoid in-memory caches from ever containing records that are not
87 present in persistent storage.
88 """
89 raise NotImplementedError()
91 @abstractmethod
92 def join(
93 self,
94 builder: QueryBuilder,
95 *,
96 regions: NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement] | None = None,
97 timespans: NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation] | None = None,
98 ) -> sqlalchemy.sql.FromClause:
99 """Add the dimension element's logical table to a query under
100 construction.
102 This is a visitor pattern interface that is expected to be called only
103 by `QueryBuilder.joinDimensionElement`.
105 Parameters
106 ----------
107 builder : `QueryBuilder`
108 Builder for the query that should contain this element.
109 regions : `NamedKeyDict`, optional
110 A mapping from `DimensionElement` to a SQLAlchemy column containing
111 the region for that element, which should be updated to include a
112 region column for this element if one exists. If `None`,
113 ``self.element`` is not being included in the query via a spatial
114 join.
115 timespan : `NamedKeyDict`, optional
116 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
117 columns containing the timespan for that element, which should be
118 updated to include timespan columns for this element if they exist.
119 If `None`, ``self.element`` is not being included in the query via
120 a temporal join.
122 Returns
123 -------
124 fromClause : `sqlalchemy.sql.FromClause`
125 Table or clause for the element which is joined.
127 Notes
128 -----
129 Elements are only included in queries via spatial and/or temporal joins
130 when necessary to connect them to other elements in the query, so
131 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
132 because an element has a region or timespan.
133 """
134 raise NotImplementedError()
136 @abstractmethod
137 def insert(self, *records: DimensionRecord, replace: bool = False, skip_existing: bool = False) -> None:
138 """Insert one or more records into storage.
140 Parameters
141 ----------
142 records
143 One or more instances of the `DimensionRecord` subclass for the
144 element this storage is associated with.
145 replace: `bool`, optional
146 If `True` (`False` is default), replace existing records in the
147 database if there is a conflict.
148 skip_existing : `bool`, optional
149 If `True` (`False` is default), skip insertion if a record with
150 the same primary key values already exists.
152 Raises
153 ------
154 TypeError
155 Raised if the element does not support record insertion.
156 sqlalchemy.exc.IntegrityError
157 Raised if one or more records violate database integrity
158 constraints.
160 Notes
161 -----
162 As `insert` is expected to be called only by a `Registry`, we rely
163 on `Registry` to provide transactionality, both by using a SQLALchemy
164 connection shared with the `Registry` and by relying on it to call
165 `clearCaches` when rolling back transactions.
166 """
167 raise NotImplementedError()
169 @abstractmethod
170 def sync(self, record: DimensionRecord, update: bool = False) -> bool | dict[str, Any]:
171 """Synchronize a record with the database, inserting it only if it does
172 not exist and comparing values if it does.
174 Parameters
175 ----------
176 record : `DimensionRecord`.
177 An instance of the `DimensionRecord` subclass for the
178 element this storage is associated with.
179 update: `bool`, optional
180 If `True` (`False` is default), update the existing record in the
181 database if there is a conflict.
183 Returns
184 -------
185 inserted_or_updated : `bool` or `dict`
186 `True` if a new row was inserted, `False` if no changes were
187 needed, or a `dict` mapping updated column names to their old
188 values if an update was performed (only possible if
189 ``update=True``).
191 Raises
192 ------
193 DatabaseConflictError
194 Raised if the record exists in the database (according to primary
195 key lookup) but is inconsistent with the given one.
196 TypeError
197 Raised if the element does not support record synchronization.
198 sqlalchemy.exc.IntegrityError
199 Raised if one or more records violate database integrity
200 constraints.
201 """
202 raise NotImplementedError()
204 @abstractmethod
205 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
206 """Retrieve records from storage.
208 Parameters
209 ----------
210 dataIds : `DataCoordinateIterable`
211 Data IDs that identify the records to be retrieved.
213 Returns
214 -------
215 records : `Iterable` [ `DimensionRecord` ]
216 Record retrieved from storage. Not all data IDs may have
217 corresponding records (if there are no records that match a data
218 ID), and even if they are, the order of inputs is not preserved.
219 """
220 raise NotImplementedError()
222 @abstractmethod
223 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
224 """Return tables used for schema digest.
226 Returns
227 -------
228 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
229 Possibly empty set of tables for schema digest calculations.
230 """
231 raise NotImplementedError()
234class GovernorDimensionRecordStorage(DimensionRecordStorage):
235 """Intermediate interface for `DimensionRecordStorage` objects that provide
236 storage for `GovernorDimension` instances.
237 """
239 @classmethod
240 @abstractmethod
241 def initialize(
242 cls,
243 db: Database,
244 dimension: GovernorDimension,
245 *,
246 context: StaticTablesContext | None = None,
247 config: Mapping[str, Any],
248 ) -> GovernorDimensionRecordStorage:
249 """Construct an instance of this class using a standardized interface.
251 Parameters
252 ----------
253 db : `Database`
254 Interface to the underlying database engine and namespace.
255 dimension : `GovernorDimension`
256 Dimension the new instance will manage records for.
257 context : `StaticTablesContext`, optional
258 If provided, an object to use to create any new tables. If not
259 provided, ``db.ensureTableExists`` should be used instead.
260 config : `Mapping`
261 Extra configuration options specific to the implementation.
263 Returns
264 -------
265 storage : `GovernorDimensionRecordStorage`
266 A new `GovernorDimensionRecordStorage` subclass instance.
267 """
268 raise NotImplementedError()
270 @property
271 @abstractmethod
272 def element(self) -> GovernorDimension:
273 # Docstring inherited from DimensionRecordStorage.
274 raise NotImplementedError()
276 @abstractmethod
277 def refresh(self) -> None:
278 """Ensure all other operations on this manager are aware of any
279 changes made by other clients since it was initialized or last
280 refreshed.
281 """
282 raise NotImplementedError()
284 @property
285 @abstractmethod
286 def values(self) -> Set[str]:
287 """All primary key values for this dimension (`set` [ `str` ]).
289 This may rely on an in-memory cache and hence not reflect changes to
290 the set of values made by other `Butler` / `Registry` clients. Call
291 `refresh` to ensure up-to-date results.
292 """
293 raise NotImplementedError()
295 @property
296 @abstractmethod
297 def table(self) -> sqlalchemy.schema.Table:
298 """The SQLAlchemy table that backs this dimension
299 (`sqlalchemy.schema.Table`).
300 """
301 raise NotImplementedError
303 @abstractmethod
304 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None:
305 """Add a function or method to be called after new records for this
306 dimension are inserted by `insert` or `sync`.
308 Parameters
309 ----------
310 callback
311 Callable that takes a single `DimensionRecord` argument. This will
312 be called immediately after any successful insertion, in the same
313 transaction.
314 """
315 raise NotImplementedError()
318class SkyPixDimensionRecordStorage(DimensionRecordStorage):
319 """Intermediate interface for `DimensionRecordStorage` objects that provide
320 storage for `SkyPixDimension` instances.
321 """
323 @property
324 @abstractmethod
325 def element(self) -> SkyPixDimension:
326 # Docstring inherited from DimensionRecordStorage.
327 raise NotImplementedError()
330class DatabaseDimensionRecordStorage(DimensionRecordStorage):
331 """Intermediate interface for `DimensionRecordStorage` objects that provide
332 storage for `DatabaseDimensionElement` instances.
333 """
335 @classmethod
336 @abstractmethod
337 def initialize(
338 cls,
339 db: Database,
340 element: DatabaseDimensionElement,
341 *,
342 context: StaticTablesContext | None = None,
343 config: Mapping[str, Any],
344 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
345 ) -> DatabaseDimensionRecordStorage:
346 """Construct an instance of this class using a standardized interface.
348 Parameters
349 ----------
350 db : `Database`
351 Interface to the underlying database engine and namespace.
352 element : `DatabaseDimensionElement`
353 Dimension element the new instance will manage records for.
354 context : `StaticTablesContext`, optional
355 If provided, an object to use to create any new tables. If not
356 provided, ``db.ensureTableExists`` should be used instead.
357 config : `Mapping`
358 Extra configuration options specific to the implementation.
359 governors : `NamedKeyMapping`
360 Mapping containing all governor dimension storage implementations.
362 Returns
363 -------
364 storage : `DatabaseDimensionRecordStorage`
365 A new `DatabaseDimensionRecordStorage` subclass instance.
366 """
367 raise NotImplementedError()
369 @property
370 @abstractmethod
371 def element(self) -> DatabaseDimensionElement:
372 # Docstring inherited from DimensionRecordStorage.
373 raise NotImplementedError()
375 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None:
376 """Inform this record storage object of the object that will manage
377 the overlaps between this element and another element.
379 This will only be called if ``self.element.spatial is not None``,
380 and will be called immediately after construction (before any other
381 methods). In the future, implementations will be required to call a
382 method on any connected overlap storage objects any time new records
383 for the element are inserted.
385 Parameters
386 ----------
387 overlaps : `DatabaseDimensionRecordStorage`
388 Object managing overlaps between this element and another
389 database-backed element.
390 """
391 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.")
394class DatabaseDimensionOverlapStorage(ABC):
395 """A base class for objects that manage overlaps between a pair of
396 database-backed dimensions.
397 """
399 @classmethod
400 @abstractmethod
401 def initialize(
402 cls,
403 db: Database,
404 elementStorage: tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage],
405 governorStorage: tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage],
406 context: StaticTablesContext | None = None,
407 ) -> DatabaseDimensionOverlapStorage:
408 """Construct an instance of this class using a standardized interface.
410 Parameters
411 ----------
412 db : `Database`
413 Interface to the underlying database engine and namespace.
414 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ]
415 Storage objects for the elements this object will related.
416 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ]
417 Storage objects for the governor dimensions of the elements this
418 object will related.
419 context : `StaticTablesContext`, optional
420 If provided, an object to use to create any new tables. If not
421 provided, ``db.ensureTableExists`` should be used instead.
423 Returns
424 -------
425 storage : `DatabaseDimensionOverlapStorage`
426 A new `DatabaseDimensionOverlapStorage` subclass instance.
427 """
428 raise NotImplementedError()
430 @property
431 @abstractmethod
432 def elements(self) -> tuple[DatabaseDimensionElement, DatabaseDimensionElement]:
433 """The pair of elements whose overlaps this object manages.
435 The order of elements is the same as their ordering within the
436 `DimensionUniverse`.
437 """
438 raise NotImplementedError()
440 @abstractmethod
441 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
442 """Return tables used for schema digest.
444 Returns
445 -------
446 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
447 Possibly empty set of tables for schema digest calculations.
448 """
449 raise NotImplementedError()
452class DimensionRecordStorageManager(VersionedExtension):
453 """An interface for managing the dimension records in a `Registry`.
455 `DimensionRecordStorageManager` primarily serves as a container and factory
456 for `DimensionRecordStorage` instances, which each provide access to the
457 records for a different `DimensionElement`.
459 Parameters
460 ----------
461 universe : `DimensionUniverse`
462 Universe of all dimensions and dimension elements known to the
463 `Registry`.
465 Notes
466 -----
467 In a multi-layer `Registry`, many dimension elements will only have
468 records in one layer (often the base layer). The union of the records
469 across all layers forms the logical table for the full `Registry`.
470 """
472 def __init__(self, *, universe: DimensionUniverse):
473 self.universe = universe
475 @classmethod
476 @abstractmethod
477 def initialize(
478 cls, db: Database, context: StaticTablesContext, *, universe: DimensionUniverse
479 ) -> DimensionRecordStorageManager:
480 """Construct an instance of the manager.
482 Parameters
483 ----------
484 db : `Database`
485 Interface to the underlying database engine and namespace.
486 context : `StaticTablesContext`
487 Context object obtained from `Database.declareStaticTables`; used
488 to declare any tables that should always be present in a layer
489 implemented with this manager.
490 universe : `DimensionUniverse`
491 Universe graph containing dimensions known to this `Registry`.
493 Returns
494 -------
495 manager : `DimensionRecordStorageManager`
496 An instance of a concrete `DimensionRecordStorageManager` subclass.
497 """
498 raise NotImplementedError()
500 @abstractmethod
501 def refresh(self) -> None:
502 """Ensure all other operations on this manager are aware of any
503 changes made by other clients since it was initialized or last
504 refreshed.
505 """
506 raise NotImplementedError()
508 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
509 """Interface to `get` that raises `LookupError` instead of returning
510 `None` on failure.
511 """
512 r = self.get(element)
513 if r is None:
514 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
515 return r
517 @abstractmethod
518 def get(self, element: DimensionElement) -> DimensionRecordStorage | None:
519 """Return an object that provides access to the records associated with
520 the given element, if one exists in this layer.
522 Parameters
523 ----------
524 element : `DimensionElement`
525 Element for which records should be returned.
527 Returns
528 -------
529 records : `DimensionRecordStorage` or `None`
530 The object representing the records for the given element in this
531 layer, or `None` if there are no records for that element in this
532 layer.
534 Notes
535 -----
536 Dimension elements registered by another client of the same layer since
537 the last call to `initialize` or `refresh` may not be found.
538 """
539 raise NotImplementedError()
541 @abstractmethod
542 def register(self, element: DimensionElement) -> DimensionRecordStorage:
543 """Ensure that this layer can hold records for the given element,
544 creating new tables as necessary.
546 Parameters
547 ----------
548 element : `DimensionElement`
549 Element for which a table should created (as necessary) and
550 an associated `DimensionRecordStorage` returned.
552 Returns
553 -------
554 records : `DimensionRecordStorage`
555 The object representing the records for the given element in this
556 layer.
558 Raises
559 ------
560 TransactionInterruption
561 Raised if this operation is invoked within a `Database.transaction`
562 context.
563 """
564 raise NotImplementedError()
566 @abstractmethod
567 def saveDimensionGraph(self, graph: DimensionGraph) -> int:
568 """Save a `DimensionGraph` definition to the database, allowing it to
569 be retrieved later via the returned key.
571 Parameters
572 ----------
573 graph : `DimensionGraph`
574 Set of dimensions to save.
576 Returns
577 -------
578 key : `int`
579 Integer used as the unique key for this `DimensionGraph` in the
580 database.
582 Raises
583 ------
584 TransactionInterruption
585 Raised if this operation is invoked within a `Database.transaction`
586 context.
587 """
588 raise NotImplementedError()
590 @abstractmethod
591 def loadDimensionGraph(self, key: int) -> DimensionGraph:
592 """Retrieve a `DimensionGraph` that was previously saved in the
593 database.
595 Parameters
596 ----------
597 key : `int`
598 Integer used as the unique key for this `DimensionGraph` in the
599 database.
601 Returns
602 -------
603 graph : `DimensionGraph`
604 Retrieved graph.
606 Raises
607 ------
608 KeyError
609 Raised if the given key cannot be found in the database.
610 """
611 raise NotImplementedError()
613 @abstractmethod
614 def clearCaches(self) -> None:
615 """Clear any in-memory caches held by nested `DimensionRecordStorage`
616 instances.
618 This is called by `Registry` when transactions are rolled back, to
619 avoid in-memory caches from ever containing records that are not
620 present in persistent storage.
621 """
622 raise NotImplementedError()
624 universe: DimensionUniverse
625 """Universe of all dimensions and dimension elements known to the
626 `Registry` (`DimensionUniverse`).
627 """