Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py : 38%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = ["DimensionRecordStorage", "DimensionRecordStorageManager"]
25from abc import ABC, abstractmethod
26from typing import Optional, Type, TYPE_CHECKING
28import sqlalchemy
30from ...core import SkyPixDimension
32if TYPE_CHECKING: 32 ↛ 33line 32 didn't jump to line 33, because the condition on line 32 was never true
33 from ...core import (
34 DataId,
35 DimensionElement,
36 DimensionRecord,
37 DimensionUniverse,
38 Timespan,
39 )
40 from ...core.utils import NamedKeyDict
41 from ..queries import QueryBuilder
42 from ._database import Database, StaticTablesContext
45class DimensionRecordStorage(ABC):
46 """An abstract base class that represents a way of storing the records
47 associated with a single `DimensionElement`.
49 Concrete `DimensionRecordStorage` instances should generally be constructed
50 via a call to `setupDimensionStorage`, which selects the appropriate
51 subclass for each element according to its configuration.
53 All `DimensionRecordStorage` methods are pure abstract, even though in some
54 cases a reasonable default implementation might be possible, in order to
55 better guarantee all methods are correctly overridden. All of these
56 potentially-defaultable implementations are extremely trivial, so asking
57 subclasses to provide them is not a significant burden.
58 """
60 @classmethod
61 @abstractmethod
62 def initialize(cls, db: Database, element: DimensionElement, *,
63 context: Optional[StaticTablesContext] = None) -> DimensionRecordStorage:
64 """Construct an instance of this class using a standardized interface.
66 Parameters
67 ----------
68 db : `Database`
69 Interface to the underlying database engine and namespace.
70 element : `DimensionElement`
71 Dimension element the new instance will manage records for.
72 context : `StaticTablesContext`, optional
73 If provided, an object to use to create any new tables. If not
74 provided, ``db.ensureTableExists`` should be used instead.
76 Returns
77 -------
78 storage : `DimensionRecordStorage`
79 A new `DimensionRecordStorage` subclass instance.
80 """
81 raise NotImplementedError()
83 @staticmethod
84 def getDefaultImplementation(element: DimensionElement, ignoreCached: bool = False
85 ) -> Type[DimensionRecordStorage]:
86 """Return the default `DimensionRecordStorage` implementation for the
87 given `DimensionElement`.
89 Parameters
90 ----------
91 element : `DimensionElement`
92 The element whose properties should be examined to determine the
93 appropriate default implementation class.
94 ignoreCached : `bool`, optional
95 If `True`, ignore `DimensionElement.cached` and always return the
96 storage implementation that would be used without caching.
98 Returns
99 -------
100 cls : `type`
101 A concrete subclass of `DimensionRecordStorage`.
103 Notes
104 -----
105 At present, these defaults are always used, but we may add support for
106 explicitly setting the class to use in configuration in the future.
107 """
108 if not ignoreCached and element.cached:
109 from ..dimensions.caching import CachingDimensionRecordStorage
110 return CachingDimensionRecordStorage
111 elif element.hasTable():
112 if element.viewOf is not None:
113 if element.spatial is not None:
114 raise NotImplementedError("Spatial view dimension storage is not supported.")
115 from ..dimensions.query import QueryDimensionRecordStorage
116 return QueryDimensionRecordStorage
117 elif element.spatial is not None:
118 from ..dimensions.spatial import SpatialDimensionRecordStorage
119 return SpatialDimensionRecordStorage
120 else:
121 from ..dimensions.table import TableDimensionRecordStorage
122 return TableDimensionRecordStorage
123 elif isinstance(element, SkyPixDimension):
124 from ..dimensions.skypix import SkyPixDimensionRecordStorage
125 return SkyPixDimensionRecordStorage
126 raise NotImplementedError(f"No default DimensionRecordStorage class for {element}.")
128 @property
129 @abstractmethod
130 def element(self) -> DimensionElement:
131 """The element whose records this instance holds (`DimensionElement`).
132 """
133 raise NotImplementedError()
135 @abstractmethod
136 def clearCaches(self) -> None:
137 """Clear any in-memory caches held by the storage instance.
139 This is called by `Registry` when transactions are rolled back, to
140 avoid in-memory caches from ever containing records that are not
141 present in persistent storage.
142 """
143 raise NotImplementedError()
145 @abstractmethod
146 def join(
147 self,
148 builder: QueryBuilder, *,
149 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None,
150 timespans: Optional[NamedKeyDict[DimensionElement, Timespan[sqlalchemy.sql.ColumnElement]]] = None,
151 ) -> sqlalchemy.sql.FromClause:
152 """Add the dimension element's logical table to a query under
153 construction.
155 This is a visitor pattern interface that is expected to be called only
156 by `QueryBuilder.joinDimensionElement`.
158 Parameters
159 ----------
160 builder : `QueryBuilder`
161 Builder for the query that should contain this element.
162 regions : `NamedKeyDict`, optional
163 A mapping from `DimensionElement` to a SQLAlchemy column containing
164 the region for that element, which should be updated to include a
165 region column for this element if one exists. If `None`,
166 ``self.element`` is not being included in the query via a spatial
167 join.
168 timespan : `NamedKeyDict`, optional
169 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
170 columns containing the timespan for that element, which should be
171 updated to include timespan columns for this element if they exist.
172 If `None`, ``self.element`` is not being included in the query via
173 a temporal join.
175 Returns
176 -------
177 fromClause : `sqlalchemy.sql.FromClause`
178 Table or clause for the element which is joined.
180 Notes
181 -----
182 Elements are only included in queries via spatial and/or temporal joins
183 when necessary to connect them to other elements in the query, so
184 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
185 because an element has a region or timespan.
186 """
187 raise NotImplementedError()
189 @abstractmethod
190 def insert(self, *records: DimensionRecord) -> None:
191 """Insert one or more records into storage.
193 Parameters
194 ----------
195 records
196 One or more instances of the `DimensionRecord` subclass for the
197 element this storage is associated with.
199 Raises
200 ------
201 TypeError
202 Raised if the element does not support record insertion.
203 sqlalchemy.exc.IntegrityError
204 Raised if one or more records violate database integrity
205 constraints.
207 Notes
208 -----
209 As `insert` is expected to be called only by a `Registry`, we rely
210 on `Registry` to provide transactionality, both by using a SQLALchemy
211 connection shared with the `Registry` and by relying on it to call
212 `clearCaches` when rolling back transactions.
213 """
214 raise NotImplementedError()
216 @abstractmethod
217 def sync(self, record: DimensionRecord) -> bool:
218 """Synchronize a record with the database, inserting it only if it does
219 not exist and comparing values if it does.
221 Parameters
222 ----------
223 record : `DimensionRecord`.
224 An instance of the `DimensionRecord` subclass for the
225 element this storage is associated with.
227 Returns
228 -------
229 inserted : `bool`
230 `True` if a new row was inserted, `False` otherwise.
232 Raises
233 ------
234 DatabaseConflictError
235 Raised if the record exists in the database (according to primary
236 key lookup) but is inconsistent with the given one.
237 TypeError
238 Raised if the element does not support record synchronization.
239 sqlalchemy.exc.IntegrityError
240 Raised if one or more records violate database integrity
241 constraints.
243 Notes
244 -----
245 This method cannot be called within transactions, as it needs to be
246 able to perform its own transaction to be concurrent.
247 """
248 raise NotImplementedError()
250 @abstractmethod
251 def fetch(self, dataId: DataId) -> Optional[DimensionRecord]:
252 """Retrieve a record from storage.
254 Parameters
255 ----------
256 dataId : `DataId`
257 A data ID that identifies the record to be retrieved. This may
258 be an informal data ID dict or a validated `DataCoordinate`.
260 Returns
261 -------
262 record : `DimensionRecord` or `None`
263 A record retrieved from storage, or `None` if there is no such
264 record.
265 """
266 raise NotImplementedError()
269class DimensionRecordStorageManager(ABC):
270 """An interface for managing the dimension records in a `Registry`.
272 `DimensionRecordStorageManager` primarily serves as a container and factory
273 for `DimensionRecordStorage` instances, which each provide access to the
274 records for a different `DimensionElement`.
276 Parameters
277 ----------
278 universe : `DimensionUniverse`
279 Universe of all dimensions and dimension elements known to the
280 `Registry`.
282 Notes
283 -----
284 In a multi-layer `Registry`, many dimension elements will only have
285 records in one layer (often the base layer). The union of the records
286 across all layers forms the logical table for the full `Registry`.
287 """
288 def __init__(self, *, universe: DimensionUniverse):
289 self.universe = universe
291 @classmethod
292 @abstractmethod
293 def initialize(cls, db: Database, context: StaticTablesContext, *,
294 universe: DimensionUniverse) -> DimensionRecordStorageManager:
295 """Construct an instance of the manager.
297 Parameters
298 ----------
299 db : `Database`
300 Interface to the underlying database engine and namespace.
301 context : `StaticTablesContext`
302 Context object obtained from `Database.declareStaticTables`; used
303 to declare any tables that should always be present in a layer
304 implemented with this manager.
305 universe : `DimensionUniverse`
306 Universe graph containing dimensions known to this `Registry`.
308 Returns
309 -------
310 manager : `DimensionRecordStorageManager`
311 An instance of a concrete `DimensionRecordStorageManager` subclass.
312 """
313 raise NotImplementedError()
315 @abstractmethod
316 def refresh(self) -> None:
317 """Ensure all other operations on this manager are aware of any
318 dataset types that may have been registered by other clients since
319 it was initialized or last refreshed.
320 """
321 raise NotImplementedError()
323 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
324 """Interface to `get` that raises `LookupError` instead of returning
325 `None` on failure.
326 """
327 r = self.get(element)
328 if r is None:
329 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
330 return r
332 @abstractmethod
333 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]:
334 """Return an object that provides access to the records associated with
335 the given element, if one exists in this layer.
337 Parameters
338 ----------
339 element : `DimensionElement`
340 Element for which records should be returned.
342 Returns
343 -------
344 records : `DimensionRecordStorage` or `None`
345 The object representing the records for the given element in this
346 layer, or `None` if there are no records for that element in this
347 layer.
349 Notes
350 -----
351 Dimension elements registered by another client of the same layer since
352 the last call to `initialize` or `refresh` may not be found.
353 """
354 raise NotImplementedError()
356 @abstractmethod
357 def register(self, element: DimensionElement) -> DimensionRecordStorage:
358 """Ensure that this layer can hold records for the given element,
359 creating new tables as necessary.
361 Parameters
362 ----------
363 element : `DimensionElement`
364 Element for which a table should created (as necessary) and
365 an associated `DimensionRecordStorage` returned.
367 Returns
368 -------
369 records : `DimensionRecordStorage`
370 The object representing the records for the given element in this
371 layer.
373 Raises
374 ------
375 TransactionInterruption
376 Raised if this operation is invoked within a `Database.transaction`
377 context.
378 """
379 raise NotImplementedError()
381 @abstractmethod
382 def clearCaches(self) -> None:
383 """Clear any in-memory caches held by nested `DimensionRecordStorage`
384 instances.
386 This is called by `Registry` when transactions are rolled back, to
387 avoid in-memory caches from ever containing records that are not
388 present in persistent storage.
389 """
390 raise NotImplementedError()
392 universe: DimensionUniverse
393 """Universe of all dimensions and dimension elements known to the
394 `Registry` (`DimensionUniverse`).
395 """