Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py : 39%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = ["DimensionRecordStorage", "DimensionRecordStorageManager"]
25from abc import ABC, abstractmethod
26from typing import Iterable, Optional, Type, TYPE_CHECKING
28import sqlalchemy
30from ...core import SkyPixDimension
31from ._versioning import VersionedExtension
33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true
34 from ...core import (
35 DatabaseTimespanRepresentation,
36 DataCoordinateIterable,
37 DimensionElement,
38 DimensionRecord,
39 DimensionUniverse,
40 NamedKeyDict,
41 )
42 from ..queries import QueryBuilder
43 from ._database import Database, StaticTablesContext
46class DimensionRecordStorage(ABC):
47 """An abstract base class that represents a way of storing the records
48 associated with a single `DimensionElement`.
50 Concrete `DimensionRecordStorage` instances should generally be constructed
51 via a call to `setupDimensionStorage`, which selects the appropriate
52 subclass for each element according to its configuration.
54 All `DimensionRecordStorage` methods are pure abstract, even though in some
55 cases a reasonable default implementation might be possible, in order to
56 better guarantee all methods are correctly overridden. All of these
57 potentially-defaultable implementations are extremely trivial, so asking
58 subclasses to provide them is not a significant burden.
59 """
61 @classmethod
62 @abstractmethod
63 def initialize(cls, db: Database, element: DimensionElement, *,
64 context: Optional[StaticTablesContext] = None) -> DimensionRecordStorage:
65 """Construct an instance of this class using a standardized interface.
67 Parameters
68 ----------
69 db : `Database`
70 Interface to the underlying database engine and namespace.
71 element : `DimensionElement`
72 Dimension element the new instance will manage records for.
73 context : `StaticTablesContext`, optional
74 If provided, an object to use to create any new tables. If not
75 provided, ``db.ensureTableExists`` should be used instead.
77 Returns
78 -------
79 storage : `DimensionRecordStorage`
80 A new `DimensionRecordStorage` subclass instance.
81 """
82 raise NotImplementedError()
84 @staticmethod
85 def getDefaultImplementation(element: DimensionElement, ignoreCached: bool = False
86 ) -> Type[DimensionRecordStorage]:
87 """Return the default `DimensionRecordStorage` implementation for the
88 given `DimensionElement`.
90 Parameters
91 ----------
92 element : `DimensionElement`
93 The element whose properties should be examined to determine the
94 appropriate default implementation class.
95 ignoreCached : `bool`, optional
96 If `True`, ignore `DimensionElement.cached` and always return the
97 storage implementation that would be used without caching.
99 Returns
100 -------
101 cls : `type`
102 A concrete subclass of `DimensionRecordStorage`.
104 Notes
105 -----
106 At present, these defaults are always used, but we may add support for
107 explicitly setting the class to use in configuration in the future.
108 """
109 if not ignoreCached and element.cached:
110 from ..dimensions.caching import CachingDimensionRecordStorage
111 return CachingDimensionRecordStorage
112 elif element.hasTable():
113 if element.viewOf is not None:
114 if element.spatial is not None:
115 raise NotImplementedError("Spatial view dimension storage is not supported.")
116 from ..dimensions.query import QueryDimensionRecordStorage
117 return QueryDimensionRecordStorage
118 elif element.spatial is not None:
119 from ..dimensions.spatial import SpatialDimensionRecordStorage
120 return SpatialDimensionRecordStorage
121 else:
122 from ..dimensions.table import TableDimensionRecordStorage
123 return TableDimensionRecordStorage
124 elif isinstance(element, SkyPixDimension):
125 from ..dimensions.skypix import SkyPixDimensionRecordStorage
126 return SkyPixDimensionRecordStorage
127 raise NotImplementedError(f"No default DimensionRecordStorage class for {element}.")
129 @property
130 @abstractmethod
131 def element(self) -> DimensionElement:
132 """The element whose records this instance holds (`DimensionElement`).
133 """
134 raise NotImplementedError()
136 @abstractmethod
137 def clearCaches(self) -> None:
138 """Clear any in-memory caches held by the storage instance.
140 This is called by `Registry` when transactions are rolled back, to
141 avoid in-memory caches from ever containing records that are not
142 present in persistent storage.
143 """
144 raise NotImplementedError()
146 @abstractmethod
147 def join(
148 self,
149 builder: QueryBuilder, *,
150 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None,
151 timespans: Optional[NamedKeyDict[DimensionElement, DatabaseTimespanRepresentation]] = None,
152 ) -> sqlalchemy.sql.FromClause:
153 """Add the dimension element's logical table to a query under
154 construction.
156 This is a visitor pattern interface that is expected to be called only
157 by `QueryBuilder.joinDimensionElement`.
159 Parameters
160 ----------
161 builder : `QueryBuilder`
162 Builder for the query that should contain this element.
163 regions : `NamedKeyDict`, optional
164 A mapping from `DimensionElement` to a SQLAlchemy column containing
165 the region for that element, which should be updated to include a
166 region column for this element if one exists. If `None`,
167 ``self.element`` is not being included in the query via a spatial
168 join.
169 timespan : `NamedKeyDict`, optional
170 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy
171 columns containing the timespan for that element, which should be
172 updated to include timespan columns for this element if they exist.
173 If `None`, ``self.element`` is not being included in the query via
174 a temporal join.
176 Returns
177 -------
178 fromClause : `sqlalchemy.sql.FromClause`
179 Table or clause for the element which is joined.
181 Notes
182 -----
183 Elements are only included in queries via spatial and/or temporal joins
184 when necessary to connect them to other elements in the query, so
185 ``regions`` and ``timespans`` cannot be assumed to be not `None` just
186 because an element has a region or timespan.
187 """
188 raise NotImplementedError()
190 @abstractmethod
191 def insert(self, *records: DimensionRecord) -> None:
192 """Insert one or more records into storage.
194 Parameters
195 ----------
196 records
197 One or more instances of the `DimensionRecord` subclass for the
198 element this storage is associated with.
200 Raises
201 ------
202 TypeError
203 Raised if the element does not support record insertion.
204 sqlalchemy.exc.IntegrityError
205 Raised if one or more records violate database integrity
206 constraints.
208 Notes
209 -----
210 As `insert` is expected to be called only by a `Registry`, we rely
211 on `Registry` to provide transactionality, both by using a SQLALchemy
212 connection shared with the `Registry` and by relying on it to call
213 `clearCaches` when rolling back transactions.
214 """
215 raise NotImplementedError()
217 @abstractmethod
218 def sync(self, record: DimensionRecord) -> bool:
219 """Synchronize a record with the database, inserting it only if it does
220 not exist and comparing values if it does.
222 Parameters
223 ----------
224 record : `DimensionRecord`.
225 An instance of the `DimensionRecord` subclass for the
226 element this storage is associated with.
228 Returns
229 -------
230 inserted : `bool`
231 `True` if a new row was inserted, `False` otherwise.
233 Raises
234 ------
235 DatabaseConflictError
236 Raised if the record exists in the database (according to primary
237 key lookup) but is inconsistent with the given one.
238 TypeError
239 Raised if the element does not support record synchronization.
240 sqlalchemy.exc.IntegrityError
241 Raised if one or more records violate database integrity
242 constraints.
244 Notes
245 -----
246 This method cannot be called within transactions, as it needs to be
247 able to perform its own transaction to be concurrent.
248 """
249 raise NotImplementedError()
251 @abstractmethod
252 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
253 """Retrieve records from storage.
255 Parameters
256 ----------
257 dataIds : `DataCoordinateIterable`
258 Data IDs that identify the records to be retrieved.
260 Returns
261 -------
262 records : `Iterable` [ `DimensionRecord` ]
263 Record retrieved from storage. Not all data IDs may have
264 corresponding records (if there are no records that match a data
265 ID), and even if they are, the order of inputs is not preserved.
266 """
267 raise NotImplementedError()
269 @abstractmethod
270 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]:
271 """Return tables used for schema digest.
273 Returns
274 -------
275 tables : `Iterable` [ `sqlalchemy.schema.Table` ]
276 Possibly empty set of tables for schema digest calculations.
277 """
278 raise NotImplementedError()
281class DimensionRecordStorageManager(VersionedExtension):
282 """An interface for managing the dimension records in a `Registry`.
284 `DimensionRecordStorageManager` primarily serves as a container and factory
285 for `DimensionRecordStorage` instances, which each provide access to the
286 records for a different `DimensionElement`.
288 Parameters
289 ----------
290 universe : `DimensionUniverse`
291 Universe of all dimensions and dimension elements known to the
292 `Registry`.
294 Notes
295 -----
296 In a multi-layer `Registry`, many dimension elements will only have
297 records in one layer (often the base layer). The union of the records
298 across all layers forms the logical table for the full `Registry`.
299 """
300 def __init__(self, *, universe: DimensionUniverse):
301 self.universe = universe
303 @classmethod
304 @abstractmethod
305 def initialize(cls, db: Database, context: StaticTablesContext, *,
306 universe: DimensionUniverse) -> DimensionRecordStorageManager:
307 """Construct an instance of the manager.
309 Parameters
310 ----------
311 db : `Database`
312 Interface to the underlying database engine and namespace.
313 context : `StaticTablesContext`
314 Context object obtained from `Database.declareStaticTables`; used
315 to declare any tables that should always be present in a layer
316 implemented with this manager.
317 universe : `DimensionUniverse`
318 Universe graph containing dimensions known to this `Registry`.
320 Returns
321 -------
322 manager : `DimensionRecordStorageManager`
323 An instance of a concrete `DimensionRecordStorageManager` subclass.
324 """
325 raise NotImplementedError()
327 @abstractmethod
328 def refresh(self) -> None:
329 """Ensure all other operations on this manager are aware of any
330 dataset types that may have been registered by other clients since
331 it was initialized or last refreshed.
332 """
333 raise NotImplementedError()
335 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage:
336 """Interface to `get` that raises `LookupError` instead of returning
337 `None` on failure.
338 """
339 r = self.get(element)
340 if r is None:
341 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.")
342 return r
344 @abstractmethod
345 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]:
346 """Return an object that provides access to the records associated with
347 the given element, if one exists in this layer.
349 Parameters
350 ----------
351 element : `DimensionElement`
352 Element for which records should be returned.
354 Returns
355 -------
356 records : `DimensionRecordStorage` or `None`
357 The object representing the records for the given element in this
358 layer, or `None` if there are no records for that element in this
359 layer.
361 Notes
362 -----
363 Dimension elements registered by another client of the same layer since
364 the last call to `initialize` or `refresh` may not be found.
365 """
366 raise NotImplementedError()
368 @abstractmethod
369 def register(self, element: DimensionElement) -> DimensionRecordStorage:
370 """Ensure that this layer can hold records for the given element,
371 creating new tables as necessary.
373 Parameters
374 ----------
375 element : `DimensionElement`
376 Element for which a table should created (as necessary) and
377 an associated `DimensionRecordStorage` returned.
379 Returns
380 -------
381 records : `DimensionRecordStorage`
382 The object representing the records for the given element in this
383 layer.
385 Raises
386 ------
387 TransactionInterruption
388 Raised if this operation is invoked within a `Database.transaction`
389 context.
390 """
391 raise NotImplementedError()
393 @abstractmethod
394 def clearCaches(self) -> None:
395 """Clear any in-memory caches held by nested `DimensionRecordStorage`
396 instances.
398 This is called by `Registry` when transactions are rolled back, to
399 avoid in-memory caches from ever containing records that are not
400 present in persistent storage.
401 """
402 raise NotImplementedError()
404 universe: DimensionUniverse
405 """Universe of all dimensions and dimension elements known to the
406 `Registry` (`DimensionUniverse`).
407 """