Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py : 69%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "DatabaseDimensionOverlapStorage",
25 "DatabaseDimensionRecordStorage",
26 "DimensionRecordStorage",
27 "DimensionRecordStorageManager",
28 "GovernorDimensionRecordStorage",
29 "SkyPixDimensionRecordStorage",
30)
32from abc import ABC, abstractmethod
33from typing import (
34 AbstractSet, Any,
35 Callable,
36 Iterable, Mapping,
37 Optional,
38 Tuple,
39 TYPE_CHECKING,
40 Union,
41)
43import sqlalchemy
45from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension
46from ._versioning import VersionedExtension
48if TYPE_CHECKING: 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true
49 from ...core import (
50 DataCoordinateIterable,
51 DimensionElement,
52 DimensionRecord,
53 DimensionUniverse,
54 NamedKeyDict,
55 NamedKeyMapping,
56 TimespanDatabaseRepresentation,
57 )
58 from ..queries import QueryBuilder
59 from ._database import Database, StaticTablesContext
62OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]]
65class DimensionRecordStorage(ABC):
66 """An abstract base class that represents a way of storing the records
67 associated with a single `DimensionElement`.
69 Concrete `DimensionRecordStorage` instances should generally be constructed
70 via a call to `setupDimensionStorage`, which selects the appropriate
71 subclass for each element according to its configuration.
73 All `DimensionRecordStorage` methods are pure abstract, even though in some
74 cases a reasonable default implementation might be possible, in order to
75 better guarantee all methods are correctly overridden. All of these
76 potentially-defaultable implementations are extremely trivial, so asking
77 subclasses to provide them is not a significant burden.
78 """
80 @property
81 @abstractmethod
82 def element(self) -> DimensionElement:
83 """The element whose records this instance holds (`DimensionElement`).
84 """
85 raise NotImplementedError()
87 @abstractmethod
88 def clearCaches(self) -> None:
89 """Clear any in-memory caches held by the storage instance.
91 This is called by `Registry` when transactions are rolled back, to
92 avoid in-memory caches from ever containing records that are not
93 present in persistent storage.
94 """
95 raise NotImplementedError()
97 @abstractmethod
98 def join(
99 self,
100 builder: QueryBuilder, *,
101 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None,
102 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None,
103 ) -> sqlalchemy.sql.FromClause:
104 """Add the dimension element's logical table to a query under
105 construction.
107 This is a visitor pattern interface that is expected to be called only
108 by `QueryBuilder.joinDimensionElement`.
110 Parameters
111 ----------
112 builder : `QueryBuilder`
113 Builder for the query that should contain this element.
114 regions : `NamedKeyDict`, optional
115 A mapping from `DimensionElement` to a SQLAlchemy column containing
116 the region for that element, which should be updated to include a
117 region column for this element if one exists. If `None`,
118 ``self.element`` is not being included in the query via a spatial
119 join.
120 timespan : `NamedKeyDict`, optional
121 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
122 columns containing the timespan for that element, which should be
123 updated to include timespan columns for this element if they exist.
124 If `None`, ``self.element`` is not being included in the query via
125 a temporal join.
127 Returns
128 -------
129 fromClause : `sqlalchemy.sql.FromClause`
130 Table or clause for the element which is joined.
132 Notes
133 -----
134 Elements are only included in queries via spatial and/or temporal joins
135 when necessary to connect them to other elements in the query, so
136 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
137 because an element has a region or timespan.
138 """
139 raise NotImplementedError()
141 @abstractmethod
142 def insert(self, *records: DimensionRecord) -> None:
143 """Insert one or more records into storage.
145 Parameters
146 ----------
147 records
148 One or more instances of the `DimensionRecord` subclass for the
149 element this storage is associated with.
151 Raises
152 ------
153 TypeError
154 Raised if the element does not support record insertion.
155 sqlalchemy.exc.IntegrityError
156 Raised if one or more records violate database integrity
157 constraints.
159 Notes
160 -----
161 As `insert` is expected to be called only by a `Registry`, we rely
162 on `Registry` to provide transactionality, both by using a SQLALchemy
163 connection shared with the `Registry` and by relying on it to call
164 `clearCaches` when rolling back transactions.
165 """
166 raise NotImplementedError()
168 @abstractmethod
169 def sync(self, record: DimensionRecord) -> bool:
170 """Synchronize a record with the database, inserting it only if it does
171 not exist and comparing values if it does.
173 Parameters
174 ----------
175 record : `DimensionRecord`.
176 An instance of the `DimensionRecord` subclass for the
177 element this storage is associated with.
179 Returns
180 -------
181 inserted : `bool`
182 `True` if a new row was inserted, `False` otherwise.
184 Raises
185 ------
186 DatabaseConflictError
187 Raised if the record exists in the database (according to primary
188 key lookup) but is inconsistent with the given one.
189 TypeError
190 Raised if the element does not support record synchronization.
191 sqlalchemy.exc.IntegrityError
192 Raised if one or more records violate database integrity
193 constraints.
194 """
195 raise NotImplementedError()
197 @abstractmethod
198 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
199 """Retrieve records from storage.
201 Parameters
202 ----------
203 dataIds : `DataCoordinateIterable`
204 Data IDs that identify the records to be retrieved.
206 Returns
207 -------
208 records : `Iterable` [ `DimensionRecord` ]
209 Record retrieved from storage. Not all data IDs may have
210 corresponding records (if there are no records that match a data
211 ID), and even if they are, the order of inputs is not preserved.
212 """
213 raise NotImplementedError()
215 @abstractmethod
216 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
217 """Return tables used for schema digest.
219 Returns
220 -------
221 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
222 Possibly empty set of tables for schema digest calculations.
223 """
224 raise NotImplementedError()
227class GovernorDimensionRecordStorage(DimensionRecordStorage):
228 """Intermediate interface for `DimensionRecordStorage` objects that provide
229 storage for `GovernorDimension` instances.
230 """
232 @classmethod
233 @abstractmethod
234 def initialize(cls, db: Database, dimension: GovernorDimension, *,
235 context: Optional[StaticTablesContext] = None,
236 config: Mapping[str, Any],
237 ) -> GovernorDimensionRecordStorage:
238 """Construct an instance of this class using a standardized interface.
240 Parameters
241 ----------
242 db : `Database`
243 Interface to the underlying database engine and namespace.
244 dimension : `GovernorDimension`
245 Dimension the new instance will manage records for.
246 context : `StaticTablesContext`, optional
247 If provided, an object to use to create any new tables. If not
248 provided, ``db.ensureTableExists`` should be used instead.
249 config : `Mapping`
250 Extra configuration options specific to the implementation.
252 Returns
253 -------
254 storage : `GovernorDimensionRecordStorage`
255 A new `GovernorDimensionRecordStorage` subclass instance.
256 """
257 raise NotImplementedError()
259 @property
260 @abstractmethod
261 def element(self) -> GovernorDimension:
262 # Docstring inherited from DimensionRecordStorage.
263 raise NotImplementedError()
265 @abstractmethod
266 def refresh(self) -> None:
267 """Ensure all other operations on this manager are aware of any
268 changes made by other clients since it was initialized or last
269 refreshed.
270 """
271 raise NotImplementedError()
273 @property
274 @abstractmethod
275 def values(self) -> AbstractSet[str]:
276 """All primary key values for this dimension (`set` [ `str` ]).
278 This may rely on an in-memory cache and hence not reflect changes to
279 the set of values made by other `Butler` / `Registry` clients. Call
280 `refresh` to ensure up-to-date results.
281 """
282 raise NotImplementedError()
284 @property
285 @abstractmethod
286 def table(self) -> sqlalchemy.schema.Table:
287 """The SQLAlchemy table that backs this dimension
288 (`sqlalchemy.schema.Table`).
289 """
290 raise NotImplementedError
292 @abstractmethod
293 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None:
294 """Add a function or method to be called after new records for this
295 dimension are inserted by `insert` or `sync`.
297 Parameters
298 ----------
299 callback
300 Callable that takes a single `DimensionRecord` argument. This will
301 be called immediately after any successful insertion, in the same
302 transaction.
303 """
304 raise NotImplementedError()
307class SkyPixDimensionRecordStorage(DimensionRecordStorage):
308 """Intermediate interface for `DimensionRecordStorage` objects that provide
309 storage for `SkyPixDimension` instances.
310 """
312 @property
313 @abstractmethod
314 def element(self) -> SkyPixDimension:
315 # Docstring inherited from DimensionRecordStorage.
316 raise NotImplementedError()
319class DatabaseDimensionRecordStorage(DimensionRecordStorage):
320 """Intermediate interface for `DimensionRecordStorage` objects that provide
321 storage for `DatabaseDimensionElement` instances.
322 """
324 @classmethod
325 @abstractmethod
326 def initialize(cls, db: Database, element: DatabaseDimensionElement, *,
327 context: Optional[StaticTablesContext] = None,
328 config: Mapping[str, Any],
329 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
330 ) -> DatabaseDimensionRecordStorage:
331 """Construct an instance of this class using a standardized interface.
333 Parameters
334 ----------
335 db : `Database`
336 Interface to the underlying database engine and namespace.
337 element : `DatabaseDimensionElement`
338 Dimension element the new instance will manage records for.
339 context : `StaticTablesContext`, optional
340 If provided, an object to use to create any new tables. If not
341 provided, ``db.ensureTableExists`` should be used instead.
342 config : `Mapping`
343 Extra configuration options specific to the implementation.
344 governors : `NamedKeyMapping`
345 Mapping containing all governor dimension storage implementations.
347 Returns
348 -------
349 storage : `DatabaseDimensionRecordStorage`
350 A new `DatabaseDimensionRecordStorage` subclass instance.
351 """
352 raise NotImplementedError()
354 @property
355 @abstractmethod
356 def element(self) -> DatabaseDimensionElement:
357 # Docstring inherited from DimensionRecordStorage.
358 raise NotImplementedError()
360 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None:
361 """Inform this record storage object of the object that will manage
362 the overlaps between this element and another element.
364 This will only be called if ``self.element.spatial is not None``,
365 and will be called immediately after construction (before any other
366 methods). In the future, implementations will be required to call a
367 method on any connected overlap storage objects any time new records
368 for the element are inserted.
370 Parameters
371 ----------
372 overlaps : `DatabaseDimensionRecordStorage`
373 Object managing overlaps between this element and another
374 database-backed element.
375 """
376 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.")
379class DatabaseDimensionOverlapStorage(ABC):
380 """A base class for objects that manage overlaps between a pair of
381 database-backed dimensions.
382 """
384 @classmethod
385 @abstractmethod
386 def initialize(
387 cls,
388 db: Database,
389 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage],
390 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage],
391 context: Optional[StaticTablesContext] = None,
392 ) -> DatabaseDimensionOverlapStorage:
393 """Construct an instance of this class using a standardized interface.
395 Parameters
396 ----------
397 db : `Database`
398 Interface to the underlying database engine and namespace.
399 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ]
400 Storage objects for the elements this object will related.
401 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ]
402 Storage objects for the governor dimensions of the elements this
403 object will related.
404 context : `StaticTablesContext`, optional
405 If provided, an object to use to create any new tables. If not
406 provided, ``db.ensureTableExists`` should be used instead.
408 Returns
409 -------
410 storage : `DatabaseDimensionOverlapStorage`
411 A new `DatabaseDimensionOverlapStorage` subclass instance.
412 """
413 raise NotImplementedError()
415 @property
416 @abstractmethod
417 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]:
418 """The pair of elements whose overlaps this object manages.
420 The order of elements is the same as their ordering within the
421 `DimensionUniverse`.
422 """
423 raise NotImplementedError()
425 @abstractmethod
426 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
427 """Return tables used for schema digest.
429 Returns
430 -------
431 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
432 Possibly empty set of tables for schema digest calculations.
433 """
434 raise NotImplementedError()
437class DimensionRecordStorageManager(VersionedExtension):
438 """An interface for managing the dimension records in a `Registry`.
440 `DimensionRecordStorageManager` primarily serves as a container and factory
441 for `DimensionRecordStorage` instances, which each provide access to the
442 records for a different `DimensionElement`.
444 Parameters
445 ----------
446 universe : `DimensionUniverse`
447 Universe of all dimensions and dimension elements known to the
448 `Registry`.
450 Notes
451 -----
452 In a multi-layer `Registry`, many dimension elements will only have
453 records in one layer (often the base layer). The union of the records
454 across all layers forms the logical table for the full `Registry`.
455 """
456 def __init__(self, *, universe: DimensionUniverse):
457 self.universe = universe
459 @classmethod
460 @abstractmethod
461 def initialize(cls, db: Database, context: StaticTablesContext, *,
462 universe: DimensionUniverse) -> DimensionRecordStorageManager:
463 """Construct an instance of the manager.
465 Parameters
466 ----------
467 db : `Database`
468 Interface to the underlying database engine and namespace.
469 context : `StaticTablesContext`
470 Context object obtained from `Database.declareStaticTables`; used
471 to declare any tables that should always be present in a layer
472 implemented with this manager.
473 universe : `DimensionUniverse`
474 Universe graph containing dimensions known to this `Registry`.
476 Returns
477 -------
478 manager : `DimensionRecordStorageManager`
479 An instance of a concrete `DimensionRecordStorageManager` subclass.
480 """
481 raise NotImplementedError()
483 @abstractmethod
484 def refresh(self) -> None:
485 """Ensure all other operations on this manager are aware of any
486 changes made by other clients since it was initialized or last
487 refreshed.
488 """
489 raise NotImplementedError()
491 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
492 """Interface to `get` that raises `LookupError` instead of returning
493 `None` on failure.
494 """
495 r = self.get(element)
496 if r is None:
497 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
498 return r
500 @abstractmethod
501 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]:
502 """Return an object that provides access to the records associated with
503 the given element, if one exists in this layer.
505 Parameters
506 ----------
507 element : `DimensionElement`
508 Element for which records should be returned.
510 Returns
511 -------
512 records : `DimensionRecordStorage` or `None`
513 The object representing the records for the given element in this
514 layer, or `None` if there are no records for that element in this
515 layer.
517 Notes
518 -----
519 Dimension elements registered by another client of the same layer since
520 the last call to `initialize` or `refresh` may not be found.
521 """
522 raise NotImplementedError()
524 @abstractmethod
525 def register(self, element: DimensionElement) -> DimensionRecordStorage:
526 """Ensure that this layer can hold records for the given element,
527 creating new tables as necessary.
529 Parameters
530 ----------
531 element : `DimensionElement`
532 Element for which a table should created (as necessary) and
533 an associated `DimensionRecordStorage` returned.
535 Returns
536 -------
537 records : `DimensionRecordStorage`
538 The object representing the records for the given element in this
539 layer.
541 Raises
542 ------
543 TransactionInterruption
544 Raised if this operation is invoked within a `Database.transaction`
545 context.
546 """
547 raise NotImplementedError()
549 @abstractmethod
550 def saveDimensionGraph(self, graph: DimensionGraph) -> int:
551 """Save a `DimensionGraph` definition to the database, allowing it to
552 be retrieved later via the returned key.
554 Parameters
555 ----------
556 graph : `DimensionGraph`
557 Set of dimensions to save.
559 Returns
560 -------
561 key : `int`
562 Integer used as the unique key for this `DimensionGraph` in the
563 database.
565 Raises
566 ------
567 TransactionInterruption
568 Raised if this operation is invoked within a `Database.transaction`
569 context.
570 """
571 raise NotImplementedError()
573 @abstractmethod
574 def loadDimensionGraph(self, key: int) -> DimensionGraph:
575 """Retrieve a `DimensionGraph` that was previously saved in the
576 database.
578 Parameters
579 ----------
580 key : `int`
581 Integer used as the unique key for this `DimensionGraph` in the
582 database.
584 Returns
585 -------
586 graph : `DimensionGraph`
587 Retrieved graph.
589 Raises
590 ------
591 KeyError
592 Raised if the given key cannot be found in the database.
593 """
594 raise NotImplementedError()
596 @abstractmethod
597 def clearCaches(self) -> None:
598 """Clear any in-memory caches held by nested `DimensionRecordStorage`
599 instances.
601 This is called by `Registry` when transactions are rolled back, to
602 avoid in-memory caches from ever containing records that are not
603 present in persistent storage.
604 """
605 raise NotImplementedError()
607 universe: DimensionUniverse
608 """Universe of all dimensions and dimension elements known to the
609 `Registry` (`DimensionUniverse`).
610 """