Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 72%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "DatabaseDimensionOverlapStorage",
25 "DatabaseDimensionRecordStorage",
26 "DimensionRecordStorage",
27 "DimensionRecordStorageManager",
28 "GovernorDimensionRecordStorage",
29 "SkyPixDimensionRecordStorage",
30)
32from abc import ABC, abstractmethod
33from typing import TYPE_CHECKING, AbstractSet, Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union
35import sqlalchemy
37from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension
38from ._versioning import VersionedExtension
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true
41 from ...core import (
42 DataCoordinateIterable,
43 DimensionElement,
44 DimensionRecord,
45 DimensionUniverse,
46 NamedKeyDict,
47 NamedKeyMapping,
48 TimespanDatabaseRepresentation,
49 )
50 from ..queries import QueryBuilder
51 from ._database import Database, StaticTablesContext
54OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]]
57class DimensionRecordStorage(ABC):
58 """An abstract base class that represents a way of storing the records
59 associated with a single `DimensionElement`.
61 Concrete `DimensionRecordStorage` instances should generally be constructed
62 via a call to `setupDimensionStorage`, which selects the appropriate
63 subclass for each element according to its configuration.
65 All `DimensionRecordStorage` methods are pure abstract, even though in some
66 cases a reasonable default implementation might be possible, in order to
67 better guarantee all methods are correctly overridden. All of these
68 potentially-defaultable implementations are extremely trivial, so asking
69 subclasses to provide them is not a significant burden.
70 """
72 @property
73 @abstractmethod
74 def element(self) -> DimensionElement:
75 """The element whose records this instance managers
76 (`DimensionElement`).
77 """
78 raise NotImplementedError()
80 @abstractmethod
81 def clearCaches(self) -> None:
82 """Clear any in-memory caches held by the storage instance.
84 This is called by `Registry` when transactions are rolled back, to
85 avoid in-memory caches from ever containing records that are not
86 present in persistent storage.
87 """
88 raise NotImplementedError()
90 @abstractmethod
91 def join(
92 self,
93 builder: QueryBuilder,
94 *,
95 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None,
96 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None,
97 ) -> sqlalchemy.sql.FromClause:
98 """Add the dimension element's logical table to a query under
99 construction.
101 This is a visitor pattern interface that is expected to be called only
102 by `QueryBuilder.joinDimensionElement`.
104 Parameters
105 ----------
106 builder : `QueryBuilder`
107 Builder for the query that should contain this element.
108 regions : `NamedKeyDict`, optional
109 A mapping from `DimensionElement` to a SQLAlchemy column containing
110 the region for that element, which should be updated to include a
111 region column for this element if one exists. If `None`,
112 ``self.element`` is not being included in the query via a spatial
113 join.
114 timespan : `NamedKeyDict`, optional
115 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
116 columns containing the timespan for that element, which should be
117 updated to include timespan columns for this element if they exist.
118 If `None`, ``self.element`` is not being included in the query via
119 a temporal join.
121 Returns
122 -------
123 fromClause : `sqlalchemy.sql.FromClause`
124 Table or clause for the element which is joined.
126 Notes
127 -----
128 Elements are only included in queries via spatial and/or temporal joins
129 when necessary to connect them to other elements in the query, so
130 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
131 because an element has a region or timespan.
132 """
133 raise NotImplementedError()
135 @abstractmethod
136 def insert(self, *records: DimensionRecord, replace: bool = False, skip_existing: bool = False) -> None:
137 """Insert one or more records into storage.
139 Parameters
140 ----------
141 records
142 One or more instances of the `DimensionRecord` subclass for the
143 element this storage is associated with.
144 replace: `bool`, optional
145 If `True` (`False` is default), replace existing records in the
146 database if there is a conflict.
147 skip_existing : `bool`, optional
148 If `True` (`False` is default), skip insertion if a record with
149 the same primary key values already exists.
151 Raises
152 ------
153 TypeError
154 Raised if the element does not support record insertion.
155 sqlalchemy.exc.IntegrityError
156 Raised if one or more records violate database integrity
157 constraints.
159 Notes
160 -----
161 As `insert` is expected to be called only by a `Registry`, we rely
162 on `Registry` to provide transactionality, both by using a SQLALchemy
163 connection shared with the `Registry` and by relying on it to call
164 `clearCaches` when rolling back transactions.
165 """
166 raise NotImplementedError()
168 @abstractmethod
169 def sync(self, record: DimensionRecord, update: bool = False) -> Union[bool, Dict[str, Any]]:
170 """Synchronize a record with the database, inserting it only if it does
171 not exist and comparing values if it does.
173 Parameters
174 ----------
175 record : `DimensionRecord`.
176 An instance of the `DimensionRecord` subclass for the
177 element this storage is associated with.
178 update: `bool`, optional
179 If `True` (`False` is default), update the existing record in the
180 database if there is a conflict.
182 Returns
183 -------
184 inserted_or_updated : `bool` or `dict`
185 `True` if a new row was inserted, `False` if no changes were
186 needed, or a `dict` mapping updated column names to their old
187 values if an update was performed (only possible if
188 ``update=True``).
190 Raises
191 ------
192 DatabaseConflictError
193 Raised if the record exists in the database (according to primary
194 key lookup) but is inconsistent with the given one.
195 TypeError
196 Raised if the element does not support record synchronization.
197 sqlalchemy.exc.IntegrityError
198 Raised if one or more records violate database integrity
199 constraints.
200 """
201 raise NotImplementedError()
203 @abstractmethod
204 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
205 """Retrieve records from storage.
207 Parameters
208 ----------
209 dataIds : `DataCoordinateIterable`
210 Data IDs that identify the records to be retrieved.
212 Returns
213 -------
214 records : `Iterable` [ `DimensionRecord` ]
215 Record retrieved from storage. Not all data IDs may have
216 corresponding records (if there are no records that match a data
217 ID), and even if they are, the order of inputs is not preserved.
218 """
219 raise NotImplementedError()
221 @abstractmethod
222 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
223 """Return tables used for schema digest.
225 Returns
226 -------
227 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
228 Possibly empty set of tables for schema digest calculations.
229 """
230 raise NotImplementedError()
233class GovernorDimensionRecordStorage(DimensionRecordStorage):
234 """Intermediate interface for `DimensionRecordStorage` objects that provide
235 storage for `GovernorDimension` instances.
236 """
238 @classmethod
239 @abstractmethod
240 def initialize(
241 cls,
242 db: Database,
243 dimension: GovernorDimension,
244 *,
245 context: Optional[StaticTablesContext] = None,
246 config: Mapping[str, Any],
247 ) -> GovernorDimensionRecordStorage:
248 """Construct an instance of this class using a standardized interface.
250 Parameters
251 ----------
252 db : `Database`
253 Interface to the underlying database engine and namespace.
254 dimension : `GovernorDimension`
255 Dimension the new instance will manage records for.
256 context : `StaticTablesContext`, optional
257 If provided, an object to use to create any new tables. If not
258 provided, ``db.ensureTableExists`` should be used instead.
259 config : `Mapping`
260 Extra configuration options specific to the implementation.
262 Returns
263 -------
264 storage : `GovernorDimensionRecordStorage`
265 A new `GovernorDimensionRecordStorage` subclass instance.
266 """
267 raise NotImplementedError()
269 @property
270 @abstractmethod
271 def element(self) -> GovernorDimension:
272 # Docstring inherited from DimensionRecordStorage.
273 raise NotImplementedError()
275 @abstractmethod
276 def refresh(self) -> None:
277 """Ensure all other operations on this manager are aware of any
278 changes made by other clients since it was initialized or last
279 refreshed.
280 """
281 raise NotImplementedError()
283 @property
284 @abstractmethod
285 def values(self) -> AbstractSet[str]:
286 """All primary key values for this dimension (`set` [ `str` ]).
288 This may rely on an in-memory cache and hence not reflect changes to
289 the set of values made by other `Butler` / `Registry` clients. Call
290 `refresh` to ensure up-to-date results.
291 """
292 raise NotImplementedError()
294 @property
295 @abstractmethod
296 def table(self) -> sqlalchemy.schema.Table:
297 """The SQLAlchemy table that backs this dimension
298 (`sqlalchemy.schema.Table`).
299 """
300 raise NotImplementedError
302 @abstractmethod
303 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None:
304 """Add a function or method to be called after new records for this
305 dimension are inserted by `insert` or `sync`.
307 Parameters
308 ----------
309 callback
310 Callable that takes a single `DimensionRecord` argument. This will
311 be called immediately after any successful insertion, in the same
312 transaction.
313 """
314 raise NotImplementedError()
317class SkyPixDimensionRecordStorage(DimensionRecordStorage):
318 """Intermediate interface for `DimensionRecordStorage` objects that provide
319 storage for `SkyPixDimension` instances.
320 """
322 @property
323 @abstractmethod
324 def element(self) -> SkyPixDimension:
325 # Docstring inherited from DimensionRecordStorage.
326 raise NotImplementedError()
329class DatabaseDimensionRecordStorage(DimensionRecordStorage):
330 """Intermediate interface for `DimensionRecordStorage` objects that provide
331 storage for `DatabaseDimensionElement` instances.
332 """
334 @classmethod
335 @abstractmethod
336 def initialize(
337 cls,
338 db: Database,
339 element: DatabaseDimensionElement,
340 *,
341 context: Optional[StaticTablesContext] = None,
342 config: Mapping[str, Any],
343 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
344 ) -> DatabaseDimensionRecordStorage:
345 """Construct an instance of this class using a standardized interface.
347 Parameters
348 ----------
349 db : `Database`
350 Interface to the underlying database engine and namespace.
351 element : `DatabaseDimensionElement`
352 Dimension element the new instance will manage records for.
353 context : `StaticTablesContext`, optional
354 If provided, an object to use to create any new tables. If not
355 provided, ``db.ensureTableExists`` should be used instead.
356 config : `Mapping`
357 Extra configuration options specific to the implementation.
358 governors : `NamedKeyMapping`
359 Mapping containing all governor dimension storage implementations.
361 Returns
362 -------
363 storage : `DatabaseDimensionRecordStorage`
364 A new `DatabaseDimensionRecordStorage` subclass instance.
365 """
366 raise NotImplementedError()
368 @property
369 @abstractmethod
370 def element(self) -> DatabaseDimensionElement:
371 # Docstring inherited from DimensionRecordStorage.
372 raise NotImplementedError()
374 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None:
375 """Inform this record storage object of the object that will manage
376 the overlaps between this element and another element.
378 This will only be called if ``self.element.spatial is not None``,
379 and will be called immediately after construction (before any other
380 methods). In the future, implementations will be required to call a
381 method on any connected overlap storage objects any time new records
382 for the element are inserted.
384 Parameters
385 ----------
386 overlaps : `DatabaseDimensionRecordStorage`
387 Object managing overlaps between this element and another
388 database-backed element.
389 """
390 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.")
393class DatabaseDimensionOverlapStorage(ABC):
394 """A base class for objects that manage overlaps between a pair of
395 database-backed dimensions.
396 """
398 @classmethod
399 @abstractmethod
400 def initialize(
401 cls,
402 db: Database,
403 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage],
404 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage],
405 context: Optional[StaticTablesContext] = None,
406 ) -> DatabaseDimensionOverlapStorage:
407 """Construct an instance of this class using a standardized interface.
409 Parameters
410 ----------
411 db : `Database`
412 Interface to the underlying database engine and namespace.
413 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ]
414 Storage objects for the elements this object will related.
415 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ]
416 Storage objects for the governor dimensions of the elements this
417 object will related.
418 context : `StaticTablesContext`, optional
419 If provided, an object to use to create any new tables. If not
420 provided, ``db.ensureTableExists`` should be used instead.
422 Returns
423 -------
424 storage : `DatabaseDimensionOverlapStorage`
425 A new `DatabaseDimensionOverlapStorage` subclass instance.
426 """
427 raise NotImplementedError()
429 @property
430 @abstractmethod
431 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]:
432 """The pair of elements whose overlaps this object manages.
434 The order of elements is the same as their ordering within the
435 `DimensionUniverse`.
436 """
437 raise NotImplementedError()
439 @abstractmethod
440 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
441 """Return tables used for schema digest.
443 Returns
444 -------
445 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
446 Possibly empty set of tables for schema digest calculations.
447 """
448 raise NotImplementedError()
451class DimensionRecordStorageManager(VersionedExtension):
452 """An interface for managing the dimension records in a `Registry`.
454 `DimensionRecordStorageManager` primarily serves as a container and factory
455 for `DimensionRecordStorage` instances, which each provide access to the
456 records for a different `DimensionElement`.
458 Parameters
459 ----------
460 universe : `DimensionUniverse`
461 Universe of all dimensions and dimension elements known to the
462 `Registry`.
464 Notes
465 -----
466 In a multi-layer `Registry`, many dimension elements will only have
467 records in one layer (often the base layer). The union of the records
468 across all layers forms the logical table for the full `Registry`.
469 """
471 def __init__(self, *, universe: DimensionUniverse):
472 self.universe = universe
474 @classmethod
475 @abstractmethod
476 def initialize(
477 cls, db: Database, context: StaticTablesContext, *, universe: DimensionUniverse
478 ) -> DimensionRecordStorageManager:
479 """Construct an instance of the manager.
481 Parameters
482 ----------
483 db : `Database`
484 Interface to the underlying database engine and namespace.
485 context : `StaticTablesContext`
486 Context object obtained from `Database.declareStaticTables`; used
487 to declare any tables that should always be present in a layer
488 implemented with this manager.
489 universe : `DimensionUniverse`
490 Universe graph containing dimensions known to this `Registry`.
492 Returns
493 -------
494 manager : `DimensionRecordStorageManager`
495 An instance of a concrete `DimensionRecordStorageManager` subclass.
496 """
497 raise NotImplementedError()
499 @abstractmethod
500 def refresh(self) -> None:
501 """Ensure all other operations on this manager are aware of any
502 changes made by other clients since it was initialized or last
503 refreshed.
504 """
505 raise NotImplementedError()
507 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
508 """Interface to `get` that raises `LookupError` instead of returning
509 `None` on failure.
510 """
511 r = self.get(element)
512 if r is None:
513 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
514 return r
516 @abstractmethod
517 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]:
518 """Return an object that provides access to the records associated with
519 the given element, if one exists in this layer.
521 Parameters
522 ----------
523 element : `DimensionElement`
524 Element for which records should be returned.
526 Returns
527 -------
528 records : `DimensionRecordStorage` or `None`
529 The object representing the records for the given element in this
530 layer, or `None` if there are no records for that element in this
531 layer.
533 Notes
534 -----
535 Dimension elements registered by another client of the same layer since
536 the last call to `initialize` or `refresh` may not be found.
537 """
538 raise NotImplementedError()
540 @abstractmethod
541 def register(self, element: DimensionElement) -> DimensionRecordStorage:
542 """Ensure that this layer can hold records for the given element,
543 creating new tables as necessary.
545 Parameters
546 ----------
547 element : `DimensionElement`
548 Element for which a table should created (as necessary) and
549 an associated `DimensionRecordStorage` returned.
551 Returns
552 -------
553 records : `DimensionRecordStorage`
554 The object representing the records for the given element in this
555 layer.
557 Raises
558 ------
559 TransactionInterruption
560 Raised if this operation is invoked within a `Database.transaction`
561 context.
562 """
563 raise NotImplementedError()
565 @abstractmethod
566 def saveDimensionGraph(self, graph: DimensionGraph) -> int:
567 """Save a `DimensionGraph` definition to the database, allowing it to
568 be retrieved later via the returned key.
570 Parameters
571 ----------
572 graph : `DimensionGraph`
573 Set of dimensions to save.
575 Returns
576 -------
577 key : `int`
578 Integer used as the unique key for this `DimensionGraph` in the
579 database.
581 Raises
582 ------
583 TransactionInterruption
584 Raised if this operation is invoked within a `Database.transaction`
585 context.
586 """
587 raise NotImplementedError()
589 @abstractmethod
590 def loadDimensionGraph(self, key: int) -> DimensionGraph:
591 """Retrieve a `DimensionGraph` that was previously saved in the
592 database.
594 Parameters
595 ----------
596 key : `int`
597 Integer used as the unique key for this `DimensionGraph` in the
598 database.
600 Returns
601 -------
602 graph : `DimensionGraph`
603 Retrieved graph.
605 Raises
606 ------
607 KeyError
608 Raised if the given key cannot be found in the database.
609 """
610 raise NotImplementedError()
612 @abstractmethod
613 def clearCaches(self) -> None:
614 """Clear any in-memory caches held by nested `DimensionRecordStorage`
615 instances.
617 This is called by `Registry` when transactions are rolled back, to
618 avoid in-memory caches from ever containing records that are not
619 present in persistent storage.
620 """
621 raise NotImplementedError()
623 universe: DimensionUniverse
624 """Universe of all dimensions and dimension elements known to the
625 `Registry` (`DimensionUniverse`).
626 """