Coverage for python/lsst/daf/butler/registry/interfaces/_database.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = [
24 "Database",
25 "ReadOnlyDatabaseError",
26 "DatabaseConflictError",
27 "StaticTablesContext",
28]
30from abc import ABC, abstractmethod
31from contextlib import contextmanager
32from typing import (
33 Any,
34 Callable,
35 Dict,
36 Iterable,
37 Iterator,
38 List,
39 Optional,
40 Sequence,
41 Set,
42 Tuple,
43)
44import warnings
46import astropy.time
47import sqlalchemy
49from ...core import ddl, time_utils
50from .._exceptions import ConflictingDefinitionError
53def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None:
54 """Test that the definition of a table in a `ddl.TableSpec` and from
55 database introspection are consistent.
57 Parameters
58 ----------
59 name : `str`
60 Name of the table (only used in error messages).
61 spec : `ddl.TableSpec`
62 Specification of the table.
63 inspection : `dict`
64 Dictionary returned by
65 `sqlalchemy.engine.reflection.Inspector.get_columns`.
67 Raises
68 ------
69 DatabaseConflictError
70 Raised if the definitions are inconsistent.
71 """
72 columnNames = [c["name"] for c in inspection]
73 if spec.fields.names != set(columnNames):
74 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; "
75 f"specification has columns {list(spec.fields.names)}, while the "
76 f"table in the database has {columnNames}.")
79class ReadOnlyDatabaseError(RuntimeError):
80 """Exception raised when a write operation is called on a read-only
81 `Database`.
82 """
85class DatabaseConflictError(ConflictingDefinitionError):
86 """Exception raised when database content (row values or schema entities)
87 are inconsistent with what this client expects.
88 """
91class StaticTablesContext:
92 """Helper class used to declare the static schema for a registry layer
93 in a database.
95 An instance of this class is returned by `Database.declareStaticTables`,
96 which should be the only way it should be constructed.
97 """
99 def __init__(self, db: Database):
100 self._db = db
101 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = []
102 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection)
103 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace))
104 self._initializers: List[Callable[[Database], None]] = []
106 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
107 """Add a new table to the schema, returning its sqlalchemy
108 representation.
110 The new table may not actually be created until the end of the
111 context created by `Database.declareStaticTables`, allowing tables
112 to be declared in any order even in the presence of foreign key
113 relationships.
114 """
115 name = self._db._mangleTableName(name)
116 if name in self._tableNames:
117 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name,
118 schema=self._db.namespace))
119 table = self._db._convertTableSpec(name, spec, self._db._metadata)
120 for foreignKeySpec in spec.foreignKeys:
121 self._foreignKeys.append(
122 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata))
123 )
124 return table
126 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]:
127 """Add a named tuple of tables to the schema, returning their
128 SQLAlchemy representations in a named tuple of the same type.
130 The new tables may not actually be created until the end of the
131 context created by `Database.declareStaticTables`, allowing tables
132 to be declared in any order even in the presence of foreign key
133 relationships.
135 Notes
136 -----
137 ``specs`` *must* be an instance of a type created by
138 `collections.namedtuple`, not just regular tuple, and the returned
139 object is guaranteed to be the same. Because `~collections.namedtuple`
140 is just a factory for `type` objects, not an actual type itself,
141 we cannot represent this with type annotations.
142 """
143 return specs._make(self.addTable(name, spec) # type: ignore
144 for name, spec in zip(specs._fields, specs)) # type: ignore
146 def addInitializer(self, initializer: Callable[[Database], None]) -> None:
147 """Add a method that does one-time initialization of a database.
149 Initialization can mean anything that changes state of a database
150 and needs to be done exactly once after database schema was created.
151 An example for that could be population of schema attributes.
153 Parameters
154 ----------
155 initializer : callable
156 Method of a single argument which is a `Database` instance.
157 """
158 self._initializers.append(initializer)
161class Database(ABC):
162 """An abstract interface that represents a particular database engine's
163 representation of a single schema/namespace/database.
165 Parameters
166 ----------
167 origin : `int`
168 An integer ID that should be used as the default for any datasets,
169 quanta, or other entities that use a (autoincrement, origin) compound
170 primary key.
171 connection : `sqlalchemy.engine.Connection`
172 The SQLAlchemy connection this `Database` wraps.
173 namespace : `str`, optional
174 Name of the schema or namespace this instance is associated with.
175 This is passed as the ``schema`` argument when constructing a
176 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to
177 avoid confusion between "schema means namespace" and "schema means
178 table definitions".
180 Notes
181 -----
182 `Database` requires all write operations to go through its special named
183 methods. Our write patterns are sufficiently simple that we don't really
184 need the full flexibility of SQL insert/update/delete syntax, and we need
185 non-standard (but common) functionality in these operations sufficiently
186 often that it seems worthwhile to provide our own generic API.
188 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via
189 their SQLAlchemy representation) to be run, as we expect these to require
190 significantly more sophistication while still being limited to standard
191 SQL.
193 `Database` itself has several underscore-prefixed attributes:
195 - ``_connection``: SQLAlchemy object representing the connection.
196 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing
197 the tables and other schema entities.
199 These are considered protected (derived classes may access them, but other
200 code should not), and read-only, aside from executing SQL via
201 ``_connection``.
202 """
204 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection,
205 namespace: Optional[str] = None):
206 self.origin = origin
207 self.namespace = namespace
208 self._connection = connection
209 self._metadata: Optional[sqlalchemy.schema.MetaData] = None
211 @classmethod
212 def makeDefaultUri(cls, root: str) -> Optional[str]:
213 """Create a default connection URI appropriate for the given root
214 directory, or `None` if there can be no such default.
215 """
216 return None
218 @classmethod
219 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None,
220 writeable: bool = True) -> Database:
221 """Construct a database from a SQLAlchemy URI.
223 Parameters
224 ----------
225 uri : `str`
226 A SQLAlchemy URI connection string.
227 origin : `int`
228 An integer ID that should be used as the default for any datasets,
229 quanta, or other entities that use a (autoincrement, origin)
230 compound primary key.
231 namespace : `str`, optional
232 A database namespace (i.e. schema) the new instance should be
233 associated with. If `None` (default), the namespace (if any) is
234 inferred from the URI.
235 writeable : `bool`, optional
236 If `True`, allow write operations on the database, including
237 ``CREATE TABLE``.
239 Returns
240 -------
241 db : `Database`
242 A new `Database` instance.
243 """
244 return cls.fromConnection(cls.connect(uri, writeable=writeable),
245 origin=origin,
246 namespace=namespace,
247 writeable=writeable)
249 @classmethod
250 @abstractmethod
251 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection:
252 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI.
254 Parameters
255 ----------
256 uri : `str`
257 A SQLAlchemy URI connection string.
258 origin : `int`
259 An integer ID that should be used as the default for any datasets,
260 quanta, or other entities that use a (autoincrement, origin)
261 compound primary key.
262 writeable : `bool`, optional
263 If `True`, allow write operations on the database, including
264 ``CREATE TABLE``.
266 Returns
267 -------
268 connection : `sqlalchemy.engine.Connection`
269 A database connection.
271 Notes
272 -----
273 Subclasses that support other ways to connect to a database are
274 encouraged to add optional arguments to their implementation of this
275 method, as long as they maintain compatibility with the base class
276 call signature.
277 """
278 raise NotImplementedError()
280 @classmethod
281 @abstractmethod
282 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int,
283 namespace: Optional[str] = None, writeable: bool = True) -> Database:
284 """Create a new `Database` from an existing
285 `sqlalchemy.engine.Connection`.
287 Parameters
288 ----------
289 connection : `sqllachemy.engine.Connection`
290 The connection for the the database. May be shared between
291 `Database` instances.
292 origin : `int`
293 An integer ID that should be used as the default for any datasets,
294 quanta, or other entities that use a (autoincrement, origin)
295 compound primary key.
296 namespace : `str`, optional
297 A different database namespace (i.e. schema) the new instance
298 should be associated with. If `None` (default), the namespace
299 (if any) is inferred from the connection.
300 writeable : `bool`, optional
301 If `True`, allow write operations on the database, including
302 ``CREATE TABLE``.
304 Returns
305 -------
306 db : `Database`
307 A new `Database` instance.
309 Notes
310 -----
311 This method allows different `Database` instances to share the same
312 connection, which is desirable when they represent different namespaces
313 can be queried together. This also ties their transaction state,
314 however; starting a transaction in any database automatically starts
315 on in all other databases.
316 """
317 raise NotImplementedError()
319 @contextmanager
320 def transaction(self, *, interrupting: bool = False) -> Iterator:
321 """Return a context manager that represents a transaction.
323 Parameters
324 ----------
325 interrupting : `bool`
326 If `True`, this transaction block needs to be able to interrupt
327 any existing one in order to yield correct behavior.
328 """
329 assert not (interrupting and self._connection.in_transaction()), (
330 "Logic error in transaction nesting: an operation that would "
331 "interrupt the active transaction context has been requested."
332 )
333 if self._connection.in_transaction():
334 trans = self._connection.begin_nested()
335 else:
336 # Use a regular (non-savepoint) transaction only for the outermost
337 # context.
338 trans = self._connection.begin()
339 try:
340 yield
341 trans.commit()
342 except BaseException:
343 trans.rollback()
344 raise
346 @contextmanager
347 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]:
348 """Return a context manager in which the database's static DDL schema
349 can be declared.
351 Parameters
352 ----------
353 create : `bool`
354 If `True`, attempt to create all tables at the end of the context.
355 If `False`, they will be assumed to already exist.
357 Returns
358 -------
359 schema : `StaticTablesContext`
360 A helper object that is used to add new tables.
362 Raises
363 ------
364 ReadOnlyDatabaseError
365 Raised if ``create`` is `True`, `Database.isWriteable` is `False`,
366 and one or more declared tables do not already exist.
368 Examples
369 --------
370 Given a `Database` instance ``db``::
372 with db.declareStaticTables(create=True) as schema:
373 schema.addTable("table1", TableSpec(...))
374 schema.addTable("table2", TableSpec(...))
376 Notes
377 -----
378 A database's static DDL schema must be declared before any dynamic
379 tables are managed via calls to `ensureTableExists` or
380 `getExistingTable`. The order in which static schema tables are added
381 inside the context block is unimportant; they will automatically be
382 sorted and added in an order consistent with their foreign key
383 relationships.
384 """
385 if create and not self.isWriteable():
386 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.")
387 self._metadata = sqlalchemy.MetaData(schema=self.namespace)
388 try:
389 context = StaticTablesContext(self)
390 yield context
391 for table, foreignKey in context._foreignKeys:
392 table.append_constraint(foreignKey)
393 if create:
394 if self.namespace is not None:
395 if self.namespace not in context._inspector.get_schema_names():
396 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace))
397 # In our tables we have columns that make use of sqlalchemy
398 # Sequence objects. There is currently a bug in sqlalchmey that
399 # causes a deprecation warning to be thrown on a property of
400 # the Sequence object when the repr for the sequence is
401 # created. Here a filter is used to catch these deprecation
402 # warnings when tables are created.
403 with warnings.catch_warnings():
404 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning)
405 self._metadata.create_all(self._connection)
406 # call all initializer methods sequentially
407 for init in context._initializers:
408 init(self)
409 except BaseException:
410 # TODO: this is potentially dangerous if we run it on
411 # pre-existing schema.
412 self._metadata.drop_all(self._connection)
413 self._metadata = None
414 raise
416 @abstractmethod
417 def isWriteable(self) -> bool:
418 """Return `True` if this database can be modified by this client.
419 """
420 raise NotImplementedError()
422 @abstractmethod
423 def __str__(self) -> str:
424 """Return a human-readable identifier for this `Database`, including
425 any namespace or schema that identifies its names within a `Registry`.
426 """
427 raise NotImplementedError()
429 def shrinkDatabaseEntityName(self, original: str) -> str:
430 """Return a version of the given name that fits within this database
431 engine's length limits for table, constraint, indexes, and sequence
432 names.
434 Implementations should not assume that simple truncation is safe,
435 because multiple long names often begin with the same prefix.
437 The default implementation simply returns the given name.
439 Parameters
440 ----------
441 original : `str`
442 The original name.
444 Returns
445 -------
446 shrunk : `str`
447 The new, possibly shortened name.
448 """
449 return original
451 def expandDatabaseEntityName(self, shrunk: str) -> str:
452 """Retrieve the original name for a database entity that was too long
453 to fit within the database engine's limits.
455 Parameters
456 ----------
457 original : `str`
458 The original name.
460 Returns
461 -------
462 shrunk : `str`
463 The new, possibly shortened name.
464 """
465 return shrunk
467 def _mangleTableName(self, name: str) -> str:
468 """Map a logical, user-visible table name to the true table name used
469 in the database.
471 The default implementation returns the given name unchanged.
473 Parameters
474 ----------
475 name : `str`
476 Input table name. Should not include a namespace (i.e. schema)
477 prefix.
479 Returns
480 -------
481 mangled : `str`
482 Mangled version of the table name (still with no namespace prefix).
484 Notes
485 -----
486 Reimplementations of this method must be idempotent - mangling an
487 already-mangled name must have no effect.
488 """
489 return name
491 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]:
492 """Create constraints based on this spec.
494 Parameters
495 ----------
496 table : `str`
497 Name of the table this column is being added to.
498 spec : `FieldSpec`
499 Specification for the field to be added.
501 Returns
502 -------
503 constraint : `list` of `sqlalchemy.CheckConstraint`
504 Constraint added for this column.
505 """
506 # By default we return no additional constraints
507 return []
509 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData,
510 **kwds: Any) -> sqlalchemy.schema.Column:
511 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`.
513 Parameters
514 ----------
515 table : `str`
516 Name of the table this column is being added to.
517 spec : `FieldSpec`
518 Specification for the field to be added.
519 metadata : `sqlalchemy.MetaData`
520 SQLAlchemy representation of the DDL schema this field's table is
521 being added to.
522 **kwds
523 Additional keyword arguments to forward to the
524 `sqlalchemy.schema.Column` constructor. This is provided to make
525 it easier for derived classes to delegate to ``super()`` while
526 making only minor changes.
528 Returns
529 -------
530 column : `sqlalchemy.schema.Column`
531 SQLAlchemy representation of the field.
532 """
533 args = [spec.name, spec.getSizedColumnType()]
534 if spec.autoincrement:
535 # Generate a sequence to use for auto incrementing for databases
536 # that do not support it natively. This will be ignored by
537 # sqlalchemy for databases that do support it.
538 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"),
539 metadata=metadata))
540 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}."
541 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey,
542 comment=spec.doc, **kwds)
544 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData,
545 **kwds: Any) -> sqlalchemy.schema.ForeignKeyConstraint:
546 """Convert a `ForeignKeySpec` to a
547 `sqlalchemy.schema.ForeignKeyConstraint`.
549 Parameters
550 ----------
551 table : `str`
552 Name of the table this foreign key is being added to.
553 spec : `ForeignKeySpec`
554 Specification for the foreign key to be added.
555 metadata : `sqlalchemy.MetaData`
556 SQLAlchemy representation of the DDL schema this constraint is
557 being added to.
558 **kwds
559 Additional keyword arguments to forward to the
560 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is
561 provided to make it easier for derived classes to delegate to
562 ``super()`` while making only minor changes.
564 Returns
565 -------
566 constraint : `sqlalchemy.schema.ForeignKeyConstraint`
567 SQLAlchemy representation of the constraint.
568 """
569 name = self.shrinkDatabaseEntityName(
570 "_".join(["fkey", table, self._mangleTableName(spec.table)]
571 + list(spec.target) + list(spec.source))
572 )
573 return sqlalchemy.schema.ForeignKeyConstraint(
574 spec.source,
575 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target],
576 name=name,
577 ondelete=spec.onDelete
578 )
580 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData,
581 **kwds: Any) -> sqlalchemy.schema.Table:
582 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`.
584 Parameters
585 ----------
586 spec : `TableSpec`
587 Specification for the foreign key to be added.
588 metadata : `sqlalchemy.MetaData`
589 SQLAlchemy representation of the DDL schema this table is being
590 added to.
591 **kwds
592 Additional keyword arguments to forward to the
593 `sqlalchemy.schema.Table` constructor. This is provided to make it
594 easier for derived classes to delegate to ``super()`` while making
595 only minor changes.
597 Returns
598 -------
599 table : `sqlalchemy.schema.Table`
600 SQLAlchemy representation of the table.
602 Notes
603 -----
604 This method does not handle ``spec.foreignKeys`` at all, in order to
605 avoid circular dependencies. These are added by higher-level logic in
606 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`.
607 """
608 name = self._mangleTableName(name)
609 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields]
611 # Add any column constraints
612 for fieldSpec in spec.fields:
613 args.extend(self._makeColumnConstraints(name, fieldSpec))
615 # Track indexes added for primary key and unique constraints, to make
616 # sure we don't add duplicate explicit or foreign key indexes for
617 # those.
618 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)}
619 args.extend(
620 sqlalchemy.schema.UniqueConstraint(
621 *columns,
622 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns)))
623 )
624 for columns in spec.unique
625 )
626 allIndexes.update(spec.unique)
627 args.extend(
628 sqlalchemy.schema.Index(
629 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))),
630 *columns,
631 unique=(columns in spec.unique)
632 )
633 for columns in spec.indexes if columns not in allIndexes
634 )
635 allIndexes.update(spec.indexes)
636 args.extend(
637 sqlalchemy.schema.Index(
638 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)),
639 *fk.source,
640 )
641 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes
642 )
643 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}."
644 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds)
646 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause:
647 """Ensure that a table with the given name and specification exists,
648 creating it if necessary.
650 Parameters
651 ----------
652 name : `str`
653 Name of the table (not including namespace qualifiers).
654 spec : `TableSpec`
655 Specification for the table. This will be used when creating the
656 table, and *may* be used when obtaining an existing table to check
657 for consistency, but no such check is guaranteed.
659 Returns
660 -------
661 table : `sqlalchemy.schema.Table`
662 SQLAlchemy representation of the table.
664 Raises
665 ------
666 ReadOnlyDatabaseError
667 Raised if `isWriteable` returns `False`, and the table does not
668 already exist.
669 DatabaseConflictError
670 Raised if the table exists but ``spec`` is inconsistent with its
671 definition.
673 Notes
674 -----
675 This method may not be called within transactions. It may be called on
676 read-only databases if and only if the table does in fact already
677 exist.
679 Subclasses may override this method, but usually should not need to.
680 """
681 assert not self._connection.in_transaction(), "Table creation interrupts transactions."
682 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
683 table = self.getExistingTable(name, spec)
684 if table is not None:
685 return table
686 if not self.isWriteable():
687 raise ReadOnlyDatabaseError(
688 f"Table {name} does not exist, and cannot be created "
689 f"because database {self} is read-only."
690 )
691 table = self._convertTableSpec(name, spec, self._metadata)
692 for foreignKeySpec in spec.foreignKeys:
693 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
694 table.create(self._connection)
695 return table
697 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]:
698 """Obtain an existing table with the given name and specification.
700 Parameters
701 ----------
702 name : `str`
703 Name of the table (not including namespace qualifiers).
704 spec : `TableSpec`
705 Specification for the table. This will be used when creating the
706 SQLAlchemy representation of the table, and it is used to
707 check that the actual table in the database is consistent.
709 Returns
710 -------
711 table : `sqlalchemy.schema.Table` or `None`
712 SQLAlchemy representation of the table, or `None` if it does not
713 exist.
715 Raises
716 ------
717 DatabaseConflictError
718 Raised if the table exists but ``spec`` is inconsistent with its
719 definition.
721 Notes
722 -----
723 This method can be called within transactions and never modifies the
724 database.
726 Subclasses may override this method, but usually should not need to.
727 """
728 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
729 name = self._mangleTableName(name)
730 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}")
731 if table is not None:
732 if spec.fields.names != set(table.columns.keys()):
733 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new "
734 f"specification has columns {list(spec.fields.names)}, while "
735 f"the previous definition has {list(table.columns.keys())}.")
736 else:
737 inspector = sqlalchemy.engine.reflection.Inspector(self._connection)
738 if name in inspector.get_table_names(schema=self.namespace):
739 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace))
740 table = self._convertTableSpec(name, spec, self._metadata)
741 for foreignKeySpec in spec.foreignKeys:
742 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
743 return table
744 return table
746 def sync(self, table: sqlalchemy.schema.Table, *,
747 keys: Dict[str, Any],
748 compared: Optional[Dict[str, Any]] = None,
749 extra: Optional[Dict[str, Any]] = None,
750 returning: Optional[Sequence[str]] = None,
751 ) -> Tuple[Optional[Dict[str, Any]], bool]:
752 """Insert into a table as necessary to ensure database contains
753 values equivalent to the given ones.
755 Parameters
756 ----------
757 table : `sqlalchemy.schema.Table`
758 Table to be queried and possibly inserted into.
759 keys : `dict`
760 Column name-value pairs used to search for an existing row; must
761 be a combination that can be used to select a single row if one
762 exists. If such a row does not exist, these values are used in
763 the insert.
764 compared : `dict`, optional
765 Column name-value pairs that are compared to those in any existing
766 row. If such a row does not exist, these rows are used in the
767 insert.
768 extra : `dict`, optional
769 Column name-value pairs that are ignored if a matching row exists,
770 but used in an insert if one is necessary.
771 returning : `~collections.abc.Sequence` of `str`, optional
772 The names of columns whose values should be returned.
774 Returns
775 -------
776 row : `dict`, optional
777 The value of the fields indicated by ``returning``, or `None` if
778 ``returning`` is `None`.
779 inserted : `bool`
780 If `True`, a new row was inserted.
782 Raises
783 ------
784 DatabaseConflictError
785 Raised if the values in ``compared`` do not match the values in the
786 database.
787 ReadOnlyDatabaseError
788 Raised if `isWriteable` returns `False`, and no matching record
789 already exists.
791 Notes
792 -----
793 This method may not be called within transactions. It may be called on
794 read-only databases if and only if the matching row does in fact
795 already exist.
796 """
798 def check() -> Tuple[int, Optional[List[str]], Optional[List]]:
799 """Query for a row that matches the ``key`` argument, and compare
800 to what was given by the caller.
802 Returns
803 -------
804 n : `int`
805 Number of matching rows. ``n != 1`` is always an error, but
806 it's a different kind of error depending on where `check` is
807 being called.
808 bad : `list` of `str`, or `None`
809 The subset of the keys of ``compared`` for which the existing
810 values did not match the given one. Once again, ``not bad``
811 is always an error, but a different kind on context. `None`
812 if ``n != 1``
813 result : `list` or `None`
814 Results in the database that correspond to the columns given
815 in ``returning``, or `None` if ``returning is None``.
816 """
817 toSelect: Set[str] = set()
818 if compared is not None:
819 toSelect.update(compared.keys())
820 if returning is not None:
821 toSelect.update(returning)
822 if not toSelect:
823 # Need to select some column, even if we just want to see
824 # how many rows we get back.
825 toSelect.add(next(iter(keys.keys())))
826 selectSql = sqlalchemy.sql.select(
827 [table.columns[k].label(k) for k in toSelect]
828 ).select_from(table).where(
829 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])
830 )
831 fetched = list(self._connection.execute(selectSql).fetchall())
832 if len(fetched) != 1:
833 return len(fetched), None, None
834 existing = fetched[0]
835 if compared is not None:
837 def safeNotEqual(a: Any, b: Any) -> bool:
838 if isinstance(a, astropy.time.Time):
839 return not time_utils.times_equal(a, b)
840 return a != b
842 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}"
843 for k, v in compared.items()
844 if safeNotEqual(existing[k], v)]
845 else:
846 inconsistencies = []
847 if returning is not None:
848 toReturn: Optional[list] = [existing[k] for k in returning]
849 else:
850 toReturn = None
851 return 1, inconsistencies, toReturn
853 if self.isWriteable():
854 # Database is writeable. Try an insert first, but allow it to fail
855 # (in only specific ways).
856 row = keys.copy()
857 if compared is not None:
858 row.update(compared)
859 if extra is not None:
860 row.update(extra)
861 insertSql = table.insert().values(row)
862 try:
863 with self.transaction(interrupting=True):
864 self._connection.execute(insertSql)
865 # Need to perform check() for this branch inside the
866 # transaction, so we roll back an insert that didn't do
867 # what we expected. That limits the extent to which we
868 # can reduce duplication between this block and the other
869 # ones that perform similar logic.
870 n, bad, result = check()
871 if n < 1:
872 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.")
873 elif n > 1:
874 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
875 f"unique constraint for table {table.name}.")
876 elif bad:
877 raise RuntimeError(
878 f"Conflict ({bad}) in sync after successful insert; this is "
879 f"possible if the same table is being updated by a concurrent "
880 f"process that isn't using sync, but it may also be a bug in "
881 f"daf_butler."
882 )
883 # No exceptions, so it looks like we inserted the requested row
884 # successfully.
885 inserted = True
886 except sqlalchemy.exc.IntegrityError as err:
887 # Most likely cause is that an equivalent row already exists,
888 # but it could also be some other constraint. Query for the
889 # row we think we matched to resolve that question.
890 n, bad, result = check()
891 if n < 1:
892 # There was no matched row; insertion failed for some
893 # completely different reason. Just re-raise the original
894 # IntegrityError.
895 raise
896 elif n > 2:
897 # There were multiple matched rows, which means we
898 # conflicted *and* the arguments were bad to begin with.
899 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
900 f"unique constraint for table {table.name}.") from err
901 elif bad:
902 # No logic bug, but data conflicted on the keys given.
903 raise DatabaseConflictError(f"Conflict in sync for table "
904 f"{table.name} on column(s) {bad}.") from err
905 # The desired row is already present and consistent with what
906 # we tried to insert.
907 inserted = False
908 else:
909 assert not self._connection.in_transaction(), (
910 "Calling sync within a transaction block is an error even "
911 "on a read-only database."
912 )
913 # Database is not writeable; just see if the row exists.
914 n, bad, result = check()
915 if n < 1:
916 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.")
917 elif n > 1:
918 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.")
919 elif bad:
920 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.")
921 inserted = False
922 if returning is None:
923 return None, inserted
924 else:
925 assert result is not None
926 return {k: v for k, v in zip(returning, result)}, inserted
928 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False,
929 ) -> Optional[List[int]]:
930 """Insert one or more rows into a table, optionally returning
931 autoincrement primary key values.
933 Parameters
934 ----------
935 table : `sqlalchemy.schema.Table`
936 Table rows should be inserted into.
937 returnIds: `bool`
938 If `True` (`False` is default), return the values of the table's
939 autoincrement primary key field (which much exist).
940 *rows
941 Positional arguments are the rows to be inserted, as dictionaries
942 mapping column name to value. The keys in all dictionaries must
943 be the same.
945 Returns
946 -------
947 ids : `None`, or `list` of `int`
948 If ``returnIds`` is `True`, a `list` containing the inserted
949 values for the table's autoincrement primary key.
951 Raises
952 ------
953 ReadOnlyDatabaseError
954 Raised if `isWriteable` returns `False` when this method is called.
956 Notes
957 -----
958 The default implementation uses bulk insert syntax when ``returnIds``
959 is `False`, and a loop over single-row insert operations when it is
960 `True`.
962 Derived classes should reimplement when they can provide a more
963 efficient implementation (especially for the latter case).
965 May be used inside transaction contexts, so implementations may not
966 perform operations that interrupt transactions.
967 """
968 if not self.isWriteable():
969 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.")
970 if not rows:
971 if returnIds:
972 return []
973 else:
974 return None
975 if not returnIds:
976 self._connection.execute(table.insert(), *rows)
977 return None
978 else:
979 sql = table.insert()
980 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows]
982 @abstractmethod
983 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None:
984 """Insert one or more rows into a table, replacing any existing rows
985 for which insertion of a new row would violate the primary key
986 constraint.
988 Parameters
989 ----------
990 table : `sqlalchemy.schema.Table`
991 Table rows should be inserted into.
992 *rows
993 Positional arguments are the rows to be inserted, as dictionaries
994 mapping column name to value. The keys in all dictionaries must
995 be the same.
997 Raises
998 ------
999 ReadOnlyDatabaseError
1000 Raised if `isWriteable` returns `False` when this method is called.
1002 Notes
1003 -----
1004 May be used inside transaction contexts, so implementations may not
1005 perform operations that interrupt transactions.
1007 Implementations should raise a `sqlalchemy.exc.IntegrityError`
1008 exception when a constraint other than the primary key would be
1009 violated.
1011 Implementations are not required to support `replace` on tables
1012 with autoincrement keys.
1013 """
1014 raise NotImplementedError()
1016 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int:
1017 """Delete one or more rows from a table.
1019 Parameters
1020 ----------
1021 table : `sqlalchemy.schema.Table`
1022 Table that rows should be deleted from.
1023 columns: `~collections.abc.Iterable` of `str`
1024 The names of columns that will be used to constrain the rows to
1025 be deleted; these will be combined via ``AND`` to form the
1026 ``WHERE`` clause of the delete query.
1027 *rows
1028 Positional arguments are the keys of rows to be deleted, as
1029 dictionaries mapping column name to value. The keys in all
1030 dictionaries must exactly the names in ``columns``.
1032 Returns
1033 -------
1034 count : `int`
1035 Number of rows deleted.
1037 Raises
1038 ------
1039 ReadOnlyDatabaseError
1040 Raised if `isWriteable` returns `False` when this method is called.
1042 Notes
1043 -----
1044 May be used inside transaction contexts, so implementations may not
1045 perform operations that interrupt transactions.
1047 The default implementation should be sufficient for most derived
1048 classes.
1049 """
1050 if not self.isWriteable():
1051 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.")
1052 if columns and not rows:
1053 # If there are no columns, this operation is supposed to delete
1054 # everything (so we proceed as usual). But if there are columns,
1055 # but no rows, it was a constrained bulk operation where the
1056 # constraint is that no rows match, and we should short-circuit
1057 # while reporting that no rows were affected.
1058 return 0
1059 sql = table.delete()
1060 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns]
1061 if whereTerms:
1062 sql = sql.where(sqlalchemy.sql.and_(*whereTerms))
1063 return self._connection.execute(sql, *rows).rowcount
1065 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int:
1066 """Update one or more rows in a table.
1068 Parameters
1069 ----------
1070 table : `sqlalchemy.schema.Table`
1071 Table containing the rows to be updated.
1072 where : `dict` [`str`, `str`]
1073 A mapping from the names of columns that will be used to search for
1074 existing rows to the keys that will hold these values in the
1075 ``rows`` dictionaries. Note that these may not be the same due to
1076 SQLAlchemy limitations.
1077 *rows
1078 Positional arguments are the rows to be updated. The keys in all
1079 dictionaries must be the same, and may correspond to either a
1080 value in the ``where`` dictionary or the name of a column to be
1081 updated.
1083 Returns
1084 -------
1085 count : `int`
1086 Number of rows matched (regardless of whether the update actually
1087 modified them).
1089 Raises
1090 ------
1091 ReadOnlyDatabaseError
1092 Raised if `isWriteable` returns `False` when this method is called.
1094 Notes
1095 -----
1096 May be used inside transaction contexts, so implementations may not
1097 perform operations that interrupt transactions.
1099 The default implementation should be sufficient for most derived
1100 classes.
1101 """
1102 if not self.isWriteable():
1103 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.")
1104 if not rows:
1105 return 0
1106 sql = table.update().where(
1107 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()])
1108 )
1109 return self._connection.execute(sql, *rows).rowcount
1111 def query(self, sql: sqlalchemy.sql.FromClause,
1112 *args: Any, **kwds: Any) -> sqlalchemy.engine.ResultProxy:
1113 """Run a SELECT query against the database.
1115 Parameters
1116 ----------
1117 sql : `sqlalchemy.sql.FromClause`
1118 A SQLAlchemy representation of a ``SELECT`` query.
1119 *args
1120 Additional positional arguments are forwarded to
1121 `sqlalchemy.engine.Connection.execute`.
1122 **kwds
1123 Additional keyword arguments are forwarded to
1124 `sqlalchemy.engine.Connection.execute`.
1126 Returns
1127 -------
1128 result : `sqlalchemy.engine.ResultProxy`
1129 Query results.
1131 Notes
1132 -----
1133 The default implementation should be sufficient for most derived
1134 classes.
1135 """
1136 # TODO: should we guard against non-SELECT queries here?
1137 return self._connection.execute(sql, *args, **kwds)
1139 origin: int
1140 """An integer ID that should be used as the default for any datasets,
1141 quanta, or other entities that use a (autoincrement, origin) compound
1142 primary key (`int`).
1143 """
1145 namespace: Optional[str]
1146 """The schema or namespace this database instance is associated with
1147 (`str` or `None`).
1148 """