Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 95%
37 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-10-11 01:27 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = ("DimensionRecordStorageManager",)
31from abc import abstractmethod
32from collections.abc import Iterable, Mapping, Set
33from typing import TYPE_CHECKING, Any
35from lsst.daf.relation import Join, Relation
37from ...dimensions import (
38 DataCoordinate,
39 DataIdValue,
40 DimensionElement,
41 DimensionGroup,
42 DimensionRecord,
43 DimensionRecordSet,
44 DimensionUniverse,
45)
46from ...dimensions.record_cache import DimensionRecordCache
47from ._versioning import VersionedExtension, VersionTuple
49if TYPE_CHECKING:
50 from ...direct_query_driver import ( # Future query system (direct,server).
51 Postprocessing,
52 SqlJoinsBuilder,
53 SqlSelectBuilder,
54 )
55 from ...queries.tree import AnyDatasetType, Predicate # Future query system (direct,client,server).
56 from .. import queries # Old Registry.query* system.
57 from ._database import Database, StaticTablesContext
60class DimensionRecordStorageManager(VersionedExtension):
61 """An interface for managing the dimension records in a `Registry`.
63 `DimensionRecordStorageManager` primarily serves as a container and factory
64 for `DimensionRecordStorage` instances, which each provide access to the
65 records for a different `DimensionElement`.
67 Parameters
68 ----------
69 universe : `DimensionUniverse`
70 Universe of all dimensions and dimension elements known to the
71 `Registry`.
72 registry_schema_version : `VersionTuple` or `None`, optional
73 Version of registry schema.
75 Notes
76 -----
77 In a multi-layer `Registry`, many dimension elements will only have
78 records in one layer (often the base layer). The union of the records
79 across all layers forms the logical table for the full `Registry`.
80 """
82 def __init__(self, *, universe: DimensionUniverse, registry_schema_version: VersionTuple | None = None):
83 super().__init__(registry_schema_version=registry_schema_version)
84 self.universe = universe
86 @abstractmethod
87 def clone(self, db: Database) -> DimensionRecordStorageManager:
88 """Make an independent copy of this manager instance bound to a new
89 `Database` instance.
91 Parameters
92 ----------
93 db : `Database`
94 New `Database` object to use when instantiating the manager.
96 Returns
97 -------
98 instance : `DatasetRecordStorageManager`
99 New manager instance with the same configuration as this instance,
100 but bound to a new Database object.
101 """
102 raise NotImplementedError()
104 @classmethod
105 @abstractmethod
106 def initialize(
107 cls,
108 db: Database,
109 context: StaticTablesContext,
110 *,
111 universe: DimensionUniverse,
112 registry_schema_version: VersionTuple | None = None,
113 ) -> DimensionRecordStorageManager:
114 """Construct an instance of the manager.
116 Parameters
117 ----------
118 db : `Database`
119 Interface to the underlying database engine and namespace.
120 context : `StaticTablesContext`
121 Context object obtained from `Database.declareStaticTables`; used
122 to declare any tables that should always be present in a layer
123 implemented with this manager.
124 universe : `DimensionUniverse`
125 Universe graph containing dimensions known to this `Registry`.
126 registry_schema_version : `VersionTuple` or `None`
127 Schema version of this extension as defined in registry.
129 Returns
130 -------
131 manager : `DimensionRecordStorageManager`
132 An instance of a concrete `DimensionRecordStorageManager` subclass.
133 """
134 raise NotImplementedError()
136 def fetch_cache_dict(self) -> dict[str, DimensionRecordSet]:
137 """Return a `dict` that can back a `DimensionRecordSet`.
139 This method is intended as the ``fetch`` callback argument to
140 `DimensionRecordCache`, in contexts where direct SQL queries are
141 possible.
142 """
143 raise NotImplementedError()
145 @abstractmethod
146 def insert(
147 self,
148 element: DimensionElement,
149 *records: DimensionRecord,
150 replace: bool = False,
151 skip_existing: bool = False,
152 ) -> None:
153 """Insert one or more records into storage.
155 Parameters
156 ----------
157 element : `DimensionElement`
158 Dimension element that provides the definition for records.
159 *records : `DimensionRecord`
160 One or more instances of the `DimensionRecord` subclass for the
161 element this storage is associated with.
162 replace : `bool`, optional
163 If `True` (`False` is default), replace existing records in the
164 database if there is a conflict.
165 skip_existing : `bool`, optional
166 If `True` (`False` is default), skip insertion if a record with
167 the same primary key values already exists.
169 Raises
170 ------
171 TypeError
172 Raised if the element does not support record insertion.
173 sqlalchemy.exc.IntegrityError
174 Raised if one or more records violate database integrity
175 constraints.
176 """
177 raise NotImplementedError()
179 @abstractmethod
180 def sync(self, record: DimensionRecord, update: bool = False) -> bool | dict[str, Any]:
181 """Synchronize a record with the database, inserting it only if it does
182 not exist and comparing values if it does.
184 Parameters
185 ----------
186 record : `DimensionRecord`
187 An instance of the `DimensionRecord` subclass for the
188 element this storage is associated with.
189 update : `bool`, optional
190 If `True` (`False` is default), update the existing record in the
191 database if there is a conflict.
193 Returns
194 -------
195 inserted_or_updated : `bool` or `dict`
196 `True` if a new row was inserted, `False` if no changes were
197 needed, or a `dict` mapping updated column names to their old
198 values if an update was performed (only possible if
199 ``update=True``).
201 Raises
202 ------
203 DatabaseConflictError
204 Raised if the record exists in the database (according to primary
205 key lookup) but is inconsistent with the given one.
206 TypeError
207 Raised if the element does not support record synchronization.
208 sqlalchemy.exc.IntegrityError
209 Raised if one or more records violate database integrity
210 constraints.
211 """
212 raise NotImplementedError()
214 @abstractmethod
215 def fetch_one(
216 self,
217 element_name: str,
218 data_id: DataCoordinate,
219 cache: DimensionRecordCache,
220 ) -> DimensionRecord | None:
221 """Retrieve a single record from storage.
223 Parameters
224 ----------
225 element_name : `str`
226 Name of the dimension element for the record to fetch.
227 data_id : `DataCoordinate`
228 Data ID of the record to fetch. Implied dimensions do not need to
229 be present.
230 cache : `DimensionRecordCache`
231 Cache to look in first.
233 Returns
234 -------
235 record : `DimensionRecord` or `None`
236 Fetched record, or *possibly* `None` if there was no match for the
237 given data ID.
238 """
239 raise NotImplementedError()
241 @abstractmethod
242 def save_dimension_group(self, group: DimensionGroup) -> int:
243 """Save a `DimensionGroup` definition to the database, allowing it to
244 be retrieved later via the returned key.
246 If this dimension group has already been saved, this method just
247 returns the key already associated with it.
249 Parameters
250 ----------
251 group : `DimensionGroup`
252 Set of dimensions to save.
254 Returns
255 -------
256 key : `int`
257 Integer used as the unique key for this `DimensionGroup` in the
258 database.
260 Raises
261 ------
262 TransactionInterruption
263 Raised if this operation is invoked within a `Database.transaction`
264 context.
265 """
266 raise NotImplementedError()
268 @abstractmethod
269 def load_dimension_group(self, key: int) -> DimensionGroup:
270 """Retrieve a `DimensionGroup` that was previously saved in the
271 database.
273 Parameters
274 ----------
275 key : `int`
276 Integer used as the unique key for this `DimensionGroup` in the
277 database.
279 Returns
280 -------
281 dimensions : `DimensionGroup`
282 Retrieved dimensions.
284 Raises
285 ------
286 KeyError
287 Raised if the given key cannot be found in the database.
288 """
289 raise NotImplementedError()
291 @abstractmethod
292 def join(
293 self,
294 element_name: str,
295 target: Relation,
296 join: Join,
297 context: queries.SqlQueryContext,
298 ) -> Relation:
299 """Join this dimension element's records to a relation.
301 Parameters
302 ----------
303 element_name : `str`
304 Name of the dimension element whose relation should be joined in.
305 target : `~lsst.daf.relation.Relation`
306 Existing relation to join to. Implementations may require that
307 this relation already include dimension key columns for this
308 dimension element and assume that dataset or spatial join relations
309 that might provide these will be included in the relation tree
310 first.
311 join : `~lsst.daf.relation.Join`
312 Join operation to use when the implementation is an actual join.
313 When a true join is being simulated by other relation operations,
314 this objects `~lsst.daf.relation.Join.min_columns` and
315 `~lsst.daf.relation.Join.max_columns` should still be respected.
316 context : `.queries.SqlQueryContext`
317 Object that manages relation engines and database-side state (e.g.
318 temporary tables) for the query.
320 Returns
321 -------
322 joined : `~lsst.daf.relation.Relation`
323 New relation that includes this relation's dimension key and record
324 columns, as well as all columns in ``target``, with rows
325 constrained to those for which this element's dimension key values
326 exist in the registry and rows already exist in ``target``.
327 """
328 raise NotImplementedError()
330 @abstractmethod
331 def make_joins_builder(self, element: DimensionElement, fields: Set[str]) -> SqlJoinsBuilder:
332 """Make a `..direct_query_driver.SqlJoinsBuilder` that represents a
333 dimension element table.
335 Parameters
336 ----------
337 element : `DimensionElement`
338 Dimension element the table corresponds to.
339 fields : `~collections.abc.Set` [ `str` ]
340 Names of fields to make available in the builder. These can be any
341 metadata or alternate key field in the element's schema, including
342 the special ``region`` and ``timespan`` fields. Dimension keys in
343 the element's schema are always included.
345 Returns
346 -------
347 builder : `..direct_query_driver.SqlJoinsBuilder`
348 A query-construction object representing a table or subquery. This
349 is guaranteed to have rows that are unique over dimension keys and
350 all possible key values for this dimension, so joining in a
351 dimension element table:
353 - never introduces duplicates into the query's result rows;
354 - never restricts the query's rows *except* to ensure
355 required-implied relationships are followed.
356 """
357 raise NotImplementedError()
359 @abstractmethod
360 def process_query_overlaps(
361 self,
362 dimensions: DimensionGroup,
363 predicate: Predicate,
364 join_operands: Iterable[DimensionGroup],
365 calibration_dataset_types: Set[str | AnyDatasetType],
366 allow_duplicates: bool,
367 constraint_data_id: Mapping[str, DataIdValue],
368 ) -> tuple[Predicate, SqlSelectBuilder, Postprocessing]:
369 """Process a query's WHERE predicate and dimensions to handle spatial
370 and temporal overlaps.
372 Parameters
373 ----------
374 dimensions : `..dimensions.DimensionGroup`
375 Full dimensions of all tables to be joined into the query (even if
376 they are not included in the query results).
377 predicate : `..queries.tree.Predicate`
378 Boolean column expression that may contain user-provided spatial
379 and/or temporal overlaps intermixed with other constraints.
380 join_operands : `~collections.abc.Iterable` [ \
381 `..dimensions.DimensionGroup` ]
382 Dimensions of tables or subqueries that are already going to be
383 joined into the query that may establish their own spatial or
384 temporal relationships (e.g. a dataset search with both ``visit``
385 and ``patch`` dimensions).
386 calibration_dataset_types : `~collections.abc.Set` [ `str` or \
387 `..queries.tree.AnyDatasetType` ]
388 The names of dataset types that have been joined into the query via
389 a search that includes at least one calibration collection.
390 allow_duplicates : `bool`
391 If set to `True` then query will be allowed to return non-distinct
392 rows.
393 constraint_data_id : `~collections.abc.Mapping` [`str`, `int` | `str`]
394 Dimension values that are known to be common to all rows in the
395 query result set.
397 Returns
398 -------
399 predicate : `..queries.tree.Predicate`
400 A version of the given predicate that preserves the overall
401 behavior of the filter while possibly rewriting overlap expressions
402 that have been partially moved into ``builder`` as some combination
403 of new nested predicates, joins, and postprocessing.
404 builder : `..direct_query_driver.SqlSelectBuilder`
405 A query-construction helper object that includes any initial joins
406 and postprocessing needed to handle overlap expression extracted
407 from the original predicate.
408 postprocessing : `Postprocessing`
409 Struct representing post-query processing to be done in Python.
411 Notes
412 -----
413 Implementations must delegate to `.queries.overlaps.OverlapsVisitor`
414 (possibly by subclassing it) to ensure "automatic" spatial and temporal
415 joins are added consistently by all query-construction implementations.
416 """
417 raise NotImplementedError()
419 universe: DimensionUniverse
420 """Universe of all dimensions and dimension elements known to the
421 `Registry` (`DimensionUniverse`).
422 """