Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py : 62%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "DatabaseDimensionOverlapStorage",
25 "DatabaseDimensionRecordStorage",
26 "DimensionRecordStorage",
27 "DimensionRecordStorageManager",
28 "GovernorDimensionRecordStorage",
29 "SkyPixDimensionRecordStorage",
30)
32from abc import ABC, abstractmethod
33from typing import (
34 AbstractSet,
35 Callable,
36 Iterable,
37 Optional,
38 Tuple,
39 TYPE_CHECKING,
40 Union,
41)
43import sqlalchemy
45from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension
46from ._versioning import VersionedExtension
48if TYPE_CHECKING: 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true
49 from ...core import (
50 Config,
51 DataCoordinateIterable,
52 DimensionElement,
53 DimensionRecord,
54 DimensionUniverse,
55 NamedKeyDict,
56 NamedKeyMapping,
57 TimespanDatabaseRepresentation,
58 )
59 from ..queries import QueryBuilder
60 from ._database import Database, StaticTablesContext
63OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]]
66class DimensionRecordStorage(ABC):
67 """An abstract base class that represents a way of storing the records
68 associated with a single `DimensionElement`.
70 Concrete `DimensionRecordStorage` instances should generally be constructed
71 via a call to `setupDimensionStorage`, which selects the appropriate
72 subclass for each element according to its configuration.
74 All `DimensionRecordStorage` methods are pure abstract, even though in some
75 cases a reasonable default implementation might be possible, in order to
76 better guarantee all methods are correctly overridden. All of these
77 potentially-defaultable implementations are extremely trivial, so asking
78 subclasses to provide them is not a significant burden.
79 """
81 @property
82 @abstractmethod
83 def element(self) -> DimensionElement:
84 """The element whose records this instance holds (`DimensionElement`).
85 """
86 raise NotImplementedError()
88 @abstractmethod
89 def clearCaches(self) -> None:
90 """Clear any in-memory caches held by the storage instance.
92 This is called by `Registry` when transactions are rolled back, to
93 avoid in-memory caches from ever containing records that are not
94 present in persistent storage.
95 """
96 raise NotImplementedError()
98 @abstractmethod
99 def join(
100 self,
101 builder: QueryBuilder, *,
102 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None,
103 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None,
104 ) -> sqlalchemy.sql.FromClause:
105 """Add the dimension element's logical table to a query under
106 construction.
108 This is a visitor pattern interface that is expected to be called only
109 by `QueryBuilder.joinDimensionElement`.
111 Parameters
112 ----------
113 builder : `QueryBuilder`
114 Builder for the query that should contain this element.
115 regions : `NamedKeyDict`, optional
116 A mapping from `DimensionElement` to a SQLAlchemy column containing
117 the region for that element, which should be updated to include a
118 region column for this element if one exists. If `None`,
119 ``self.element`` is not being included in the query via a spatial
120 join.
121 timespan : `NamedKeyDict`, optional
122 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
123 columns containing the timespan for that element, which should be
124 updated to include timespan columns for this element if they exist.
125 If `None`, ``self.element`` is not being included in the query via
126 a temporal join.
128 Returns
129 -------
130 fromClause : `sqlalchemy.sql.FromClause`
131 Table or clause for the element which is joined.
133 Notes
134 -----
135 Elements are only included in queries via spatial and/or temporal joins
136 when necessary to connect them to other elements in the query, so
137 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
138 because an element has a region or timespan.
139 """
140 raise NotImplementedError()
142 @abstractmethod
143 def insert(self, *records: DimensionRecord) -> None:
144 """Insert one or more records into storage.
146 Parameters
147 ----------
148 records
149 One or more instances of the `DimensionRecord` subclass for the
150 element this storage is associated with.
152 Raises
153 ------
154 TypeError
155 Raised if the element does not support record insertion.
156 sqlalchemy.exc.IntegrityError
157 Raised if one or more records violate database integrity
158 constraints.
160 Notes
161 -----
162 As `insert` is expected to be called only by a `Registry`, we rely
163 on `Registry` to provide transactionality, both by using a SQLALchemy
164 connection shared with the `Registry` and by relying on it to call
165 `clearCaches` when rolling back transactions.
166 """
167 raise NotImplementedError()
169 @abstractmethod
170 def sync(self, record: DimensionRecord) -> bool:
171 """Synchronize a record with the database, inserting it only if it does
172 not exist and comparing values if it does.
174 Parameters
175 ----------
176 record : `DimensionRecord`.
177 An instance of the `DimensionRecord` subclass for the
178 element this storage is associated with.
180 Returns
181 -------
182 inserted : `bool`
183 `True` if a new row was inserted, `False` otherwise.
185 Raises
186 ------
187 DatabaseConflictError
188 Raised if the record exists in the database (according to primary
189 key lookup) but is inconsistent with the given one.
190 TypeError
191 Raised if the element does not support record synchronization.
192 sqlalchemy.exc.IntegrityError
193 Raised if one or more records violate database integrity
194 constraints.
195 """
196 raise NotImplementedError()
198 @abstractmethod
199 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
200 """Retrieve records from storage.
202 Parameters
203 ----------
204 dataIds : `DataCoordinateIterable`
205 Data IDs that identify the records to be retrieved.
207 Returns
208 -------
209 records : `Iterable` [ `DimensionRecord` ]
210 Record retrieved from storage. Not all data IDs may have
211 corresponding records (if there are no records that match a data
212 ID), and even if they are, the order of inputs is not preserved.
213 """
214 raise NotImplementedError()
216 @abstractmethod
217 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
218 """Return tables used for schema digest.
220 Returns
221 -------
222 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
223 Possibly empty set of tables for schema digest calculations.
224 """
225 raise NotImplementedError()
228class GovernorDimensionRecordStorage(DimensionRecordStorage):
229 """Intermediate interface for `DimensionRecordStorage` objects that provide
230 storage for `GovernorDimension` instances.
231 """
233 @classmethod
234 @abstractmethod
235 def initialize(cls, db: Database, dimension: GovernorDimension, *,
236 context: Optional[StaticTablesContext] = None,
237 config: Config,
238 ) -> GovernorDimensionRecordStorage:
239 """Construct an instance of this class using a standardized interface.
241 Parameters
242 ----------
243 db : `Database`
244 Interface to the underlying database engine and namespace.
245 dimension : `GovernorDimension`
246 Dimension the new instance will manage records for.
247 context : `StaticTablesContext`, optional
248 If provided, an object to use to create any new tables. If not
249 provided, ``db.ensureTableExists`` should be used instead.
250 config : `Config`
251 Extra configuration options specific to the implementation.
253 Returns
254 -------
255 storage : `GovernorDimensionRecordStorage`
256 A new `GovernorDimensionRecordStorage` subclass instance.
257 """
258 raise NotImplementedError()
260 @property
261 @abstractmethod
262 def element(self) -> GovernorDimension:
263 # Docstring inherited from DimensionRecordStorage.
264 raise NotImplementedError()
266 @abstractmethod
267 def refresh(self) -> None:
268 """Ensure all other operations on this manager are aware of any
269 changes made by other clients since it was initialized or last
270 refreshed.
271 """
272 raise NotImplementedError()
274 @property
275 @abstractmethod
276 def values(self) -> AbstractSet[str]:
277 """All primary key values for this dimension (`set` [ `str` ]).
279 This may rely on an in-memory cache and hence not reflect changes to
280 the set of values made by other `Butler` / `Registry` clients. Call
281 `refresh` to ensure up-to-date results.
282 """
283 raise NotImplementedError()
285 @property
286 @abstractmethod
287 def table(self) -> sqlalchemy.schema.Table:
288 """The SQLAlchemy table that backs this dimension
289 (`sqlalchemy.schema.Table`).
290 """
291 raise NotImplementedError
293 @abstractmethod
294 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None:
295 """Add a function or method to be called after new records for this
296 dimension are inserted by `insert` or `sync`.
298 Parameters
299 ----------
300 callback
301 Callable that takes a single `DimensionRecord` argument. This will
302 be called immediately after any successful insertion, in the same
303 transaction.
304 """
305 raise NotImplementedError()
308class SkyPixDimensionRecordStorage(DimensionRecordStorage):
309 """Intermediate interface for `DimensionRecordStorage` objects that provide
310 storage for `SkyPixDimension` instances.
311 """
313 @property
314 @abstractmethod
315 def element(self) -> SkyPixDimension:
316 # Docstring inherited from DimensionRecordStorage.
317 raise NotImplementedError()
320class DatabaseDimensionRecordStorage(DimensionRecordStorage):
321 """Intermediate interface for `DimensionRecordStorage` objects that provide
322 storage for `DatabaseDimensionElement` instances.
323 """
325 @classmethod
326 @abstractmethod
327 def initialize(cls, db: Database, element: DatabaseDimensionElement, *,
328 context: Optional[StaticTablesContext] = None,
329 config: Config,
330 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage],
331 ) -> DatabaseDimensionRecordStorage:
332 """Construct an instance of this class using a standardized interface.
334 Parameters
335 ----------
336 db : `Database`
337 Interface to the underlying database engine and namespace.
338 element : `DatabaseDimensionElement`
339 Dimension element the new instance will manage records for.
340 context : `StaticTablesContext`, optional
341 If provided, an object to use to create any new tables. If not
342 provided, ``db.ensureTableExists`` should be used instead.
343 config : `Config`
344 Extra configuration options specific to the implementation.
345 governors : `NamedKeyMapping`
346 Mapping containing all governor dimension storage implementations.
348 Returns
349 -------
350 storage : `DatabaseDimensionRecordStorage`
351 A new `DatabaseDimensionRecordStorage` subclass instance.
352 """
353 raise NotImplementedError()
355 @property
356 @abstractmethod
357 def element(self) -> DatabaseDimensionElement:
358 # Docstring inherited from DimensionRecordStorage.
359 raise NotImplementedError()
361 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None:
362 """Inform this record storage object of the object that will manage
363 the overlaps between this element and another element.
365 This will only be called if ``self.element.spatial is not None``,
366 and will be called immediately after construction (before any other
367 methods). In the future, implementations will be required to call a
368 method on any connected overlap storage objects any time new records
369 for the element are inserted.
371 Parameters
372 ----------
373 overlaps : `DatabaseDimensionRecordStorage`
374 Object managing overlaps between this element and another
375 database-backed element.
376 """
377 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.")
380class DatabaseDimensionOverlapStorage(ABC):
381 """A base class for objects that manage overlaps between a pair of
382 database-backed dimensions.
383 """
385 @classmethod
386 @abstractmethod
387 def initialize(
388 cls,
389 db: Database,
390 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage],
391 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage],
392 context: Optional[StaticTablesContext] = None,
393 ) -> DatabaseDimensionOverlapStorage:
394 """Construct an instance of this class using a standardized interface.
396 Parameters
397 ----------
398 db : `Database`
399 Interface to the underlying database engine and namespace.
400 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ]
401 Storage objects for the elements this object will related.
402 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ]
403 Storage objects for the governor dimensions of the elements this
404 object will related.
405 context : `StaticTablesContext`, optional
406 If provided, an object to use to create any new tables. If not
407 provided, ``db.ensureTableExists`` should be used instead.
409 Returns
410 -------
411 storage : `DatabaseDimensionOverlapStorage`
412 A new `DatabaseDimensionOverlapStorage` subclass instance.
413 """
414 raise NotImplementedError()
416 @property
417 @abstractmethod
418 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]:
419 """The pair of elements whose overlaps this object manages.
421 The order of elements is the same as their ordering within the
422 `DimensionUniverse`.
423 """
424 raise NotImplementedError()
426 @abstractmethod
427 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
428 """Return tables used for schema digest.
430 Returns
431 -------
432 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
433 Possibly empty set of tables for schema digest calculations.
434 """
435 raise NotImplementedError()
438class DimensionRecordStorageManager(VersionedExtension):
439 """An interface for managing the dimension records in a `Registry`.
441 `DimensionRecordStorageManager` primarily serves as a container and factory
442 for `DimensionRecordStorage` instances, which each provide access to the
443 records for a different `DimensionElement`.
445 Parameters
446 ----------
447 universe : `DimensionUniverse`
448 Universe of all dimensions and dimension elements known to the
449 `Registry`.
451 Notes
452 -----
453 In a multi-layer `Registry`, many dimension elements will only have
454 records in one layer (often the base layer). The union of the records
455 across all layers forms the logical table for the full `Registry`.
456 """
457 def __init__(self, *, universe: DimensionUniverse):
458 self.universe = universe
460 @classmethod
461 @abstractmethod
462 def initialize(cls, db: Database, context: StaticTablesContext, *,
463 universe: DimensionUniverse) -> DimensionRecordStorageManager:
464 """Construct an instance of the manager.
466 Parameters
467 ----------
468 db : `Database`
469 Interface to the underlying database engine and namespace.
470 context : `StaticTablesContext`
471 Context object obtained from `Database.declareStaticTables`; used
472 to declare any tables that should always be present in a layer
473 implemented with this manager.
474 universe : `DimensionUniverse`
475 Universe graph containing dimensions known to this `Registry`.
477 Returns
478 -------
479 manager : `DimensionRecordStorageManager`
480 An instance of a concrete `DimensionRecordStorageManager` subclass.
481 """
482 raise NotImplementedError()
484 @abstractmethod
485 def refresh(self) -> None:
486 """Ensure all other operations on this manager are aware of any
487 changes made by other clients since it was initialized or last
488 refreshed.
489 """
490 raise NotImplementedError()
492 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
493 """Interface to `get` that raises `LookupError` instead of returning
494 `None` on failure.
495 """
496 r = self.get(element)
497 if r is None:
498 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
499 return r
501 @abstractmethod
502 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]:
503 """Return an object that provides access to the records associated with
504 the given element, if one exists in this layer.
506 Parameters
507 ----------
508 element : `DimensionElement`
509 Element for which records should be returned.
511 Returns
512 -------
513 records : `DimensionRecordStorage` or `None`
514 The object representing the records for the given element in this
515 layer, or `None` if there are no records for that element in this
516 layer.
518 Notes
519 -----
520 Dimension elements registered by another client of the same layer since
521 the last call to `initialize` or `refresh` may not be found.
522 """
523 raise NotImplementedError()
525 @abstractmethod
526 def register(self, element: DimensionElement) -> DimensionRecordStorage:
527 """Ensure that this layer can hold records for the given element,
528 creating new tables as necessary.
530 Parameters
531 ----------
532 element : `DimensionElement`
533 Element for which a table should created (as necessary) and
534 an associated `DimensionRecordStorage` returned.
536 Returns
537 -------
538 records : `DimensionRecordStorage`
539 The object representing the records for the given element in this
540 layer.
542 Raises
543 ------
544 TransactionInterruption
545 Raised if this operation is invoked within a `Database.transaction`
546 context.
547 """
548 raise NotImplementedError()
550 @abstractmethod
551 def saveDimensionGraph(self, graph: DimensionGraph) -> int:
552 """Save a `DimensionGraph` definition to the database, allowing it to
553 be retrieved later via the returned key.
555 Parameters
556 ----------
557 graph : `DimensionGraph`
558 Set of dimensions to save.
560 Returns
561 -------
562 key : `int`
563 Integer used as the unique key for this `DimensionGraph` in the
564 database.
566 Raises
567 ------
568 TransactionInterruption
569 Raised if this operation is invoked within a `Database.transaction`
570 context.
571 """
572 raise NotImplementedError()
574 @abstractmethod
575 def loadDimensionGraph(self, key: int) -> DimensionGraph:
576 """Retrieve a `DimensionGraph` that was previously saved in the
577 database.
579 Parameters
580 ----------
581 key : `int`
582 Integer used as the unique key for this `DimensionGraph` in the
583 database.
585 Returns
586 -------
587 graph : `DimensionGraph`
588 Retrieved graph.
590 Raises
591 ------
592 KeyError
593 Raised if the given key cannot be found in the database.
594 """
595 raise NotImplementedError()
597 @abstractmethod
598 def clearCaches(self) -> None:
599 """Clear any in-memory caches held by nested `DimensionRecordStorage`
600 instances.
602 This is called by `Registry` when transactions are rolled back, to
603 avoid in-memory caches from ever containing records that are not
604 present in persistent storage.
605 """
606 raise NotImplementedError()
608 universe: DimensionUniverse
609 """Universe of all dimensions and dimension elements known to the
610 `Registry` (`DimensionUniverse`).
611 """