Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 72%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "DatabaseDimensionOverlapStorage",
25 "DatabaseDimensionRecordStorage",
26 "DimensionRecordStorage",
27 "DimensionRecordStorageManager",
28 "GovernorDimensionRecordStorage",
29 "SkyPixDimensionRecordStorage",
30)
32from abc import ABC, abstractmethod
33from typing import TYPE_CHECKING, AbstractSet, Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union
35import sqlalchemy
37from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension
38from ._versioning import VersionedExtension
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true
41 from ...core import (
42 DataCoordinateIterable,
43 DimensionElement,
44 DimensionRecord,
45 DimensionUniverse,
46 NamedKeyDict,
47 NamedKeyMapping,
48 TimespanDatabaseRepresentation,
49 )
50 from ..queries import QueryBuilder
51 from ._database import Database, StaticTablesContext
54OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]]
57class DimensionRecordStorage(ABC):
58 """An abstract base class that represents a way of storing the records
59 associated with a single `DimensionElement`.
61 Concrete `DimensionRecordStorage` instances should generally be constructed
62 via a call to `setupDimensionStorage`, which selects the appropriate
63 subclass for each element according to its configuration.
65 All `DimensionRecordStorage` methods are pure abstract, even though in some
66 cases a reasonable default implementation might be possible, in order to
67 better guarantee all methods are correctly overridden. All of these
68 potentially-defaultable implementations are extremely trivial, so asking
69 subclasses to provide them is not a significant burden.
70 """
72 @property
73 @abstractmethod
74 def element(self) -> DimensionElement:
75 """The element whose records this instance managers
76 (`DimensionElement`).
77 """
78 raise NotImplementedError()
80 @abstractmethod
81 def clearCaches(self) -> None:
82 """Clear any in-memory caches held by the storage instance.
84 This is called by `Registry` when transactions are rolled back, to
85 avoid in-memory caches from ever containing records that are not
86 present in persistent storage.
87 """
88 raise NotImplementedError()
90 @abstractmethod
91 def join(
92 self,
93 builder: QueryBuilder,
94 *,
95 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None,
96 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None,
97 ) -> sqlalchemy.sql.FromClause:
98 """Add the dimension element's logical table to a query under
99 construction.
101 This is a visitor pattern interface that is expected to be called only
102 by `QueryBuilder.joinDimensionElement`.
104 Parameters
105 ----------
106 builder : `QueryBuilder`
107 Builder for the query that should contain this element.
108 regions : `NamedKeyDict`, optional
109 A mapping from `DimensionElement` to a SQLAlchemy column containing
110 the region for that element, which should be updated to include a
111 region column for this element if one exists. If `None`,
112 ``self.element`` is not being included in the query via a spatial
113 join.
114 timespan : `NamedKeyDict`, optional
115 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
116 columns containing the timespan for that element, which should be
117 updated to include timespan columns for this element if they exist.
118 If `None`, ``self.element`` is not being included in the query via
119 a temporal join.
121 Returns
122 -------
123 fromClause : `sqlalchemy.sql.FromClause`
124 Table or clause for the element which is joined.
126 Notes
127 -----
128 Elements are only included in queries via spatial and/or temporal joins
129 when necessary to connect them to other elements in the query, so
130 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
131 because an element has a region or timespan.
132 """
133 raise NotImplementedError()
135 @abstractmethod
136 def insert(self, *records: DimensionRecord, replace: bool = False) -> None:
137 """Insert one or more records into storage.
139 Parameters
140 ----------
141 records
142 One or more instances of the `DimensionRecord` subclass for the
143 element this storage is associated with.
144 replace: `bool`, optional
145 If `True` (`False` is default), replace existing records in the
146 database if there is a conflict.
148 Raises
149 ------
150 TypeError
151 Raised if the element does not support record insertion.
152 sqlalchemy.exc.IntegrityError
153 Raised if one or more records violate database integrity
154 constraints.
156 Notes
157 -----
158 As `insert` is expected to be called only by a `Registry`, we rely
159 on `Registry` to provide transactionality, both by using a SQLALchemy
160 connection shared with the `Registry` and by relying on it to call
161 `clearCaches` when rolling back transactions.
162 """
163 raise NotImplementedError()
165 @abstractmethod
166 def sync(self, record: DimensionRecord, update: bool = False) -> Union[bool, Dict[str, Any]]:
167 """Synchronize a record with the database, inserting it only if it does
168 not exist and comparing values if it does.
170 Parameters
171 ----------
172 record : `DimensionRecord`.
173 An instance of the `DimensionRecord` subclass for the
174 element this storage is associated with.
175 update: `bool`, optional
176 If `True` (`False` is default), update the existing record in the
177 database if there is a conflict.
179 Returns
180 -------
181 inserted_or_updated : `bool` or `dict`
182 `True` if a new row was inserted, `False` if no changes were
183 needed, or a `dict` mapping updated column names to their old
184 values if an update was performed (only possible if
185 ``update=True``).
187 Raises
188 ------
189 DatabaseConflictError
190 Raised if the record exists in the database (according to primary
191 key lookup) but is inconsistent with the given one.
192 TypeError
193 Raised if the element does not support record synchronization.
194 sqlalchemy.exc.IntegrityError
195 Raised if one or more records violate database integrity
196 constraints.
197 """
198 raise NotImplementedError()
200 @abstractmethod
201 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
202 """Retrieve records from storage.
204 Parameters
205 ----------
206 dataIds : `DataCoordinateIterable`
207 Data IDs that identify the records to be retrieved.
209 Returns
210 -------
211 records : `Iterable` [ `DimensionRecord` ]
212 Record retrieved from storage. Not all data IDs may have
213 corresponding records (if there are no records that match a data
214 ID), and even if they are, the order of inputs is not preserved.
215 """
216 raise NotImplementedError()
218 @abstractmethod
219 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
220 """Return tables used for schema digest.
222 Returns
223 -------
224 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
225 Possibly empty set of tables for schema digest calculations.
226 """
227 raise NotImplementedError()
230class GovernorDimensionRecordStorage(DimensionRecordStorage):
231 """Intermediate interface for `DimensionRecordStorage` objects that provide
232 storage for `GovernorDimension` instances.
233 """
235 @classmethod
236 @abstractmethod
237 def initialize(
238 cls,
239 db: Database,
240 dimension: GovernorDimension,
241 *,
242 context: Optional[StaticTablesContext] = None,
243 config: Mapping[str, Any],
244 ) -> GovernorDimensionRecordStorage:
245 """Construct an instance of this class using a standardized interface.
247 Parameters
248 ----------
249 db : `Database`
250 Interface to the underlying database engine and namespace.
251 dimension : `GovernorDimension`
252 Dimension the new instance will manage records for.
253 context : `StaticTablesContext`, optional
254 If provided, an object to use to create any new tables. If not
255 provided, ``db.ensureTableExists`` should be used instead.
256 config : `Mapping`
257 Extra configuration options specific to the implementation.
259 Returns
260 -------
261 storage : `GovernorDimensionRecordStorage`
262 A new `GovernorDimensionRecordStorage` subclass instance.
263 """
264 raise NotImplementedError()
266 @property
267 @abstractmethod
268 def element(self) -> GovernorDimension:
269 # Docstring inherited from DimensionRecordStorage.
270 raise NotImplementedError()
272 @abstractmethod
273 def refresh(self) -> None:
274 """Ensure all other operations on this manager are aware of any
275 changes made by other clients since it was initialized or last
276 refreshed.
277 """
278 raise NotImplementedError()
280 @property
281 @abstractmethod
282 def values(self) -> AbstractSet[str]:
283 """All primary key values for this dimension (`set` [ `str` ]).
285 This may rely on an in-memory cache and hence not reflect changes to
286 the set of values made by other `Butler` / `Registry` clients. Call
287 `refresh` to ensure up-to-date results.
288 """
289 raise NotImplementedError()
291 @property
292 @abstractmethod
293 def table(self) -> sqlalchemy.schema.Table:
294 """The SQLAlchemy table that backs this dimension
295 (`sqlalchemy.schema.Table`).
296 """
297 raise NotImplementedError
299 @abstractmethod
300 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None:
301 """Add a function or method to be called after new records for this
302 dimension are inserted by `insert` or `sync`.
304 Parameters
305 ----------
306 callback
307 Callable that takes a single `DimensionRecord` argument. This will
308 be called immediately after any successful insertion, in the same
309 transaction.
310 """
311 raise NotImplementedError()
314class SkyPixDimensionRecordStorage(DimensionRecordStorage):
315 """Intermediate interface for `DimensionRecordStorage` objects that provide
316 storage for `SkyPixDimension` instances.
317 """
319 @property
320 @abstractmethod
321 def element(self) -> SkyPixDimension:
322 # Docstring inherited from DimensionRecordStorage.
323 raise NotImplementedError()
326class DatabaseDimensionRecordStorage(DimensionRecordStorage):
327 """Intermediate interface for `DimensionRecordStorage` objects that provide
328 storage for `DatabaseDimensionElement` instances.
329 """
331 @classmethod
332 @abstractmethod
333 def initialize(
334 cls,
335 db: Database,
336 element: DatabaseDimensionElement,
337 *,
338 context: Optional[StaticTablesContext] = None,
339 config: Mapping[str, Any],
340 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
341 ) -> DatabaseDimensionRecordStorage:
342 """Construct an instance of this class using a standardized interface.
344 Parameters
345 ----------
346 db : `Database`
347 Interface to the underlying database engine and namespace.
348 element : `DatabaseDimensionElement`
349 Dimension element the new instance will manage records for.
350 context : `StaticTablesContext`, optional
351 If provided, an object to use to create any new tables. If not
352 provided, ``db.ensureTableExists`` should be used instead.
353 config : `Mapping`
354 Extra configuration options specific to the implementation.
355 governors : `NamedKeyMapping`
356 Mapping containing all governor dimension storage implementations.
358 Returns
359 -------
360 storage : `DatabaseDimensionRecordStorage`
361 A new `DatabaseDimensionRecordStorage` subclass instance.
362 """
363 raise NotImplementedError()
365 @property
366 @abstractmethod
367 def element(self) -> DatabaseDimensionElement:
368 # Docstring inherited from DimensionRecordStorage.
369 raise NotImplementedError()
371 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None:
372 """Inform this record storage object of the object that will manage
373 the overlaps between this element and another element.
375 This will only be called if ``self.element.spatial is not None``,
376 and will be called immediately after construction (before any other
377 methods). In the future, implementations will be required to call a
378 method on any connected overlap storage objects any time new records
379 for the element are inserted.
381 Parameters
382 ----------
383 overlaps : `DatabaseDimensionRecordStorage`
384 Object managing overlaps between this element and another
385 database-backed element.
386 """
387 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.")
390class DatabaseDimensionOverlapStorage(ABC):
391 """A base class for objects that manage overlaps between a pair of
392 database-backed dimensions.
393 """
395 @classmethod
396 @abstractmethod
397 def initialize(
398 cls,
399 db: Database,
400 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage],
401 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage],
402 context: Optional[StaticTablesContext] = None,
403 ) -> DatabaseDimensionOverlapStorage:
404 """Construct an instance of this class using a standardized interface.
406 Parameters
407 ----------
408 db : `Database`
409 Interface to the underlying database engine and namespace.
410 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ]
411 Storage objects for the elements this object will related.
412 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ]
413 Storage objects for the governor dimensions of the elements this
414 object will related.
415 context : `StaticTablesContext`, optional
416 If provided, an object to use to create any new tables. If not
417 provided, ``db.ensureTableExists`` should be used instead.
419 Returns
420 -------
421 storage : `DatabaseDimensionOverlapStorage`
422 A new `DatabaseDimensionOverlapStorage` subclass instance.
423 """
424 raise NotImplementedError()
426 @property
427 @abstractmethod
428 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]:
429 """The pair of elements whose overlaps this object manages.
431 The order of elements is the same as their ordering within the
432 `DimensionUniverse`.
433 """
434 raise NotImplementedError()
436 @abstractmethod
437 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
438 """Return tables used for schema digest.
440 Returns
441 -------
442 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
443 Possibly empty set of tables for schema digest calculations.
444 """
445 raise NotImplementedError()
448class DimensionRecordStorageManager(VersionedExtension):
449 """An interface for managing the dimension records in a `Registry`.
451 `DimensionRecordStorageManager` primarily serves as a container and factory
452 for `DimensionRecordStorage` instances, which each provide access to the
453 records for a different `DimensionElement`.
455 Parameters
456 ----------
457 universe : `DimensionUniverse`
458 Universe of all dimensions and dimension elements known to the
459 `Registry`.
461 Notes
462 -----
463 In a multi-layer `Registry`, many dimension elements will only have
464 records in one layer (often the base layer). The union of the records
465 across all layers forms the logical table for the full `Registry`.
466 """
468 def __init__(self, *, universe: DimensionUniverse):
469 self.universe = universe
471 @classmethod
472 @abstractmethod
473 def initialize(
474 cls, db: Database, context: StaticTablesContext, *, universe: DimensionUniverse
475 ) -> DimensionRecordStorageManager:
476 """Construct an instance of the manager.
478 Parameters
479 ----------
480 db : `Database`
481 Interface to the underlying database engine and namespace.
482 context : `StaticTablesContext`
483 Context object obtained from `Database.declareStaticTables`; used
484 to declare any tables that should always be present in a layer
485 implemented with this manager.
486 universe : `DimensionUniverse`
487 Universe graph containing dimensions known to this `Registry`.
489 Returns
490 -------
491 manager : `DimensionRecordStorageManager`
492 An instance of a concrete `DimensionRecordStorageManager` subclass.
493 """
494 raise NotImplementedError()
496 @abstractmethod
497 def refresh(self) -> None:
498 """Ensure all other operations on this manager are aware of any
499 changes made by other clients since it was initialized or last
500 refreshed.
501 """
502 raise NotImplementedError()
504 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
505 """Interface to `get` that raises `LookupError` instead of returning
506 `None` on failure.
507 """
508 r = self.get(element)
509 if r is None:
510 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
511 return r
513 @abstractmethod
514 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]:
515 """Return an object that provides access to the records associated with
516 the given element, if one exists in this layer.
518 Parameters
519 ----------
520 element : `DimensionElement`
521 Element for which records should be returned.
523 Returns
524 -------
525 records : `DimensionRecordStorage` or `None`
526 The object representing the records for the given element in this
527 layer, or `None` if there are no records for that element in this
528 layer.
530 Notes
531 -----
532 Dimension elements registered by another client of the same layer since
533 the last call to `initialize` or `refresh` may not be found.
534 """
535 raise NotImplementedError()
537 @abstractmethod
538 def register(self, element: DimensionElement) -> DimensionRecordStorage:
539 """Ensure that this layer can hold records for the given element,
540 creating new tables as necessary.
542 Parameters
543 ----------
544 element : `DimensionElement`
545 Element for which a table should created (as necessary) and
546 an associated `DimensionRecordStorage` returned.
548 Returns
549 -------
550 records : `DimensionRecordStorage`
551 The object representing the records for the given element in this
552 layer.
554 Raises
555 ------
556 TransactionInterruption
557 Raised if this operation is invoked within a `Database.transaction`
558 context.
559 """
560 raise NotImplementedError()
562 @abstractmethod
563 def saveDimensionGraph(self, graph: DimensionGraph) -> int:
564 """Save a `DimensionGraph` definition to the database, allowing it to
565 be retrieved later via the returned key.
567 Parameters
568 ----------
569 graph : `DimensionGraph`
570 Set of dimensions to save.
572 Returns
573 -------
574 key : `int`
575 Integer used as the unique key for this `DimensionGraph` in the
576 database.
578 Raises
579 ------
580 TransactionInterruption
581 Raised if this operation is invoked within a `Database.transaction`
582 context.
583 """
584 raise NotImplementedError()
586 @abstractmethod
587 def loadDimensionGraph(self, key: int) -> DimensionGraph:
588 """Retrieve a `DimensionGraph` that was previously saved in the
589 database.
591 Parameters
592 ----------
593 key : `int`
594 Integer used as the unique key for this `DimensionGraph` in the
595 database.
597 Returns
598 -------
599 graph : `DimensionGraph`
600 Retrieved graph.
602 Raises
603 ------
604 KeyError
605 Raised if the given key cannot be found in the database.
606 """
607 raise NotImplementedError()
609 @abstractmethod
610 def clearCaches(self) -> None:
611 """Clear any in-memory caches held by nested `DimensionRecordStorage`
612 instances.
614 This is called by `Registry` when transactions are rolled back, to
615 avoid in-memory caches from ever containing records that are not
616 present in persistent storage.
617 """
618 raise NotImplementedError()
620 universe: DimensionUniverse
621 """Universe of all dimensions and dimension elements known to the
622 `Registry` (`DimensionUniverse`).
623 """