Coverage for python/lsst/daf/butler/registry/interfaces/_database.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = [
24 "Database",
25 "ReadOnlyDatabaseError",
26 "DatabaseConflictError",
27 "StaticTablesContext",
28]
30from abc import ABC, abstractmethod
31from contextlib import contextmanager
32from typing import (
33 Any,
34 Dict,
35 Iterable,
36 List,
37 Optional,
38 Sequence,
39 Tuple,
40)
41import warnings
43import sqlalchemy
45from ...core import ddl
48def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: Dict[str, Any]):
49 """Test that the definition of a table in a `ddl.TableSpec` and from
50 database introspection are consistent.
52 Parameters
53 ----------
54 name : `str`
55 Name of the table (only used in error messages).
56 spec : `ddl.TableSpec`
57 Specification of the table.
58 inspection : `dict`
59 Dictionary returned by
60 `sqlalchemy.engine.reflection.Inspector.get_columns`.
62 Raises
63 ------
64 DatabaseConflictError
65 Raised if the definitions are inconsistent.
66 """
67 columnNames = [c["name"] for c in inspection]
68 if spec.fields.names != set(columnNames):
69 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; "
70 f"specification has columns {list(spec.fields.names)}, while the "
71 f"table in the database has {columnNames}.")
74class ReadOnlyDatabaseError(RuntimeError):
75 """Exception raised when a write operation is called on a read-only
76 `Database`.
77 """
80class DatabaseConflictError(RuntimeError):
81 """Exception raised when database content (row values or schema entities)
82 are inconsistent with what this client expects.
83 """
86class StaticTablesContext:
87 """Helper class used to declare the static schema for a registry layer
88 in a database.
90 An instance of this class is returned by `Database.declareStaticTables`,
91 which should be the only way it should be constructed.
92 """
94 def __init__(self, db: Database):
95 self._db = db
96 self._foreignKeys = []
97 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection)
98 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace))
100 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
101 """Add a new table to the schema, returning its sqlalchemy
102 representation.
104 The new table may not actually be created until the end of the
105 context created by `Database.declareStaticTables`, allowing tables
106 to be declared in any order even in the presence of foreign key
107 relationships.
108 """
109 name = self._db._mangleTableName(name)
110 if name in self._tableNames:
111 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name,
112 schema=self._db.namespace))
113 table = self._db._convertTableSpec(name, spec, self._db._metadata)
114 for foreignKeySpec in spec.foreignKeys:
115 self._foreignKeys.append(
116 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata))
117 )
118 return table
120 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]:
121 """Add a named tuple of tables to the schema, returning their
122 SQLAlchemy representations in a named tuple of the same type.
124 The new tables may not actually be created until the end of the
125 context created by `Database.declareStaticTables`, allowing tables
126 to be declared in any order even in the presence of foreign key
127 relationships.
129 Notes
130 -----
131 ``specs`` *must* be an instance of a type created by
132 `collections.namedtuple`, not just regular tuple, and the returned
133 object is guaranteed to be the same. Because `~collections.namedtuple`
134 is just a factory for `type` objects, not an actual type itself,
135 we cannot represent this with type annotations.
136 """
137 return specs._make(self.addTable(name, spec) for name, spec in zip(specs._fields, specs))
140class Database(ABC):
141 """An abstract interface that represents a particular database engine's
142 representation of a single schema/namespace/database.
144 Parameters
145 ----------
146 origin : `int`
147 An integer ID that should be used as the default for any datasets,
148 quanta, or other entities that use a (autoincrement, origin) compound
149 primary key.
150 connection : `sqlalchemy.engine.Connection`
151 The SQLAlchemy connection this `Database` wraps.
152 namespace : `str`, optional
153 Name of the schema or namespace this instance is associated with.
154 This is passed as the ``schema`` argument when constructing a
155 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to
156 avoid confusion between "schema means namespace" and "schema means
157 table definitions".
159 Notes
160 -----
161 `Database` requires all write operations to go through its special named
162 methods. Our write patterns are sufficiently simple that we don't really
163 need the full flexibility of SQL insert/update/delete syntax, and we need
164 non-standard (but common) functionality in these operations sufficiently
165 often that it seems worthwhile to provide our own generic API.
167 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via
168 their SQLAlchemy representation) to be run, as we expect these to require
169 significantly more sophistication while still being limited to standard
170 SQL.
172 `Database` itself has several underscore-prefixed attributes:
174 - ``_cs``: SQLAlchemy objects representing the connection and transaction
175 state.
176 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing
177 the tables and other schema entities.
179 These are considered protected (derived classes may access them, but other
180 code should not), and read-only, aside from executing SQL via
181 ``_connection``.
182 """
184 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection,
185 namespace: Optional[str] = None):
186 self.origin = origin
187 self.namespace = namespace
188 self._connection = connection
189 self._metadata = None
191 @classmethod
192 def makeDefaultUri(cls, root: str) -> Optional[str]:
193 """Create a default connection URI appropriate for the given root
194 directory, or `None` if there can be no such default.
195 """
196 return None
198 @classmethod
199 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None,
200 writeable: bool = True) -> Database:
201 """Construct a database from a SQLAlchemy URI.
203 Parameters
204 ----------
205 uri : `str`
206 A SQLAlchemy URI connection string.
207 origin : `int`
208 An integer ID that should be used as the default for any datasets,
209 quanta, or other entities that use a (autoincrement, origin)
210 compound primary key.
211 namespace : `str`, optional
212 A database namespace (i.e. schema) the new instance should be
213 associated with. If `None` (default), the namespace (if any) is
214 inferred from the URI.
215 writeable : `bool`, optional
216 If `True`, allow write operations on the database, including
217 ``CREATE TABLE``.
219 Returns
220 -------
221 db : `Database`
222 A new `Database` instance.
223 """
224 return cls.fromConnection(cls.connect(uri, writeable=writeable),
225 origin=origin,
226 namespace=namespace,
227 writeable=writeable)
229 @classmethod
230 @abstractmethod
231 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection:
232 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI.
234 Parameters
235 ----------
236 uri : `str`
237 A SQLAlchemy URI connection string.
238 origin : `int`
239 An integer ID that should be used as the default for any datasets,
240 quanta, or other entities that use a (autoincrement, origin)
241 compound primary key.
242 writeable : `bool`, optional
243 If `True`, allow write operations on the database, including
244 ``CREATE TABLE``.
246 Returns
247 -------
248 connection : `sqlalchemy.engine.Connection`
249 A database connection.
251 Notes
252 -----
253 Subclasses that support other ways to connect to a database are
254 encouraged to add optional arguments to their implementation of this
255 method, as long as they maintain compatibility with the base class
256 call signature.
257 """
258 raise NotImplementedError()
260 @classmethod
261 @abstractmethod
262 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int,
263 namespace: Optional[str] = None, writeable: bool = True) -> Database:
264 """Create a new `Database` from an existing
265 `sqlalchemy.engine.Connection`.
267 Parameters
268 ----------
269 connection : `sqllachemy.engine.Connection`
270 The connection for the the database. May be shared between
271 `Database` instances.
272 origin : `int`
273 An integer ID that should be used as the default for any datasets,
274 quanta, or other entities that use a (autoincrement, origin)
275 compound primary key.
276 namespace : `str`, optional
277 A different database namespace (i.e. schema) the new instance
278 should be associated with. If `None` (default), the namespace
279 (if any) is inferred from the connection.
280 writeable : `bool`, optional
281 If `True`, allow write operations on the database, including
282 ``CREATE TABLE``.
284 Returns
285 -------
286 db : `Database`
287 A new `Database` instance.
289 Notes
290 -----
291 This method allows different `Database` instances to share the same
292 connection, which is desirable when they represent different namespaces
293 can be queried together. This also ties their transaction state,
294 however; starting a transaction in any database automatically starts
295 on in all other databases.
296 """
297 raise NotImplementedError()
299 @contextmanager
300 def transaction(self, *, interrupting: bool = False) -> None:
301 """Return a context manager that represents a transaction.
303 Parameters
304 ----------
305 interrupting : `bool`
306 If `True`, this transaction block needs to be able to interrupt
307 any existing one in order to yield correct behavior.
308 """
309 assert not (interrupting and self._connection.in_transaction()), (
310 "Logic error in transaction nesting: an operation that would "
311 "interrupt the active transaction context has been requested."
312 )
313 if self._connection.in_transaction():
314 trans = self._connection.begin_nested()
315 else:
316 # Use a regular (non-savepoint) transaction only for the outermost
317 # context.
318 trans = self._connection.begin()
319 try:
320 yield
321 trans.commit()
322 except BaseException:
323 trans.rollback()
324 raise
326 @contextmanager
327 def declareStaticTables(self, *, create: bool) -> StaticTablesContext:
328 """Return a context manager in which the database's static DDL schema
329 can be declared.
331 Parameters
332 ----------
333 create : `bool`
334 If `True`, attempt to create all tables at the end of the context.
335 If `False`, they will be assumed to already exist.
337 Returns
338 -------
339 schema : `StaticTablesContext`
340 A helper object that is used to add new tables.
342 Raises
343 ------
344 ReadOnlyDatabaseError
345 Raised if ``create`` is `True`, `Database.isWriteable` is `False`,
346 and one or more declared tables do not already exist.
348 Examples
349 --------
350 Given a `Database` instance ``db``::
352 with db.declareStaticTables(create=True) as schema:
353 schema.addTable("table1", TableSpec(...))
354 schema.addTable("table2", TableSpec(...))
356 Notes
357 -----
358 A database's static DDL schema must be declared before any dynamic
359 tables are managed via calls to `ensureTableExists` or
360 `getExistingTable`. The order in which static schema tables are added
361 inside the context block is unimportant; they will automatically be
362 sorted and added in an order consistent with their foreign key
363 relationships.
364 """
365 if create and not self.isWriteable():
366 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.")
367 self._metadata = sqlalchemy.MetaData(schema=self.namespace)
368 try:
369 context = StaticTablesContext(self)
370 yield context
371 for table, foreignKey in context._foreignKeys:
372 table.append_constraint(foreignKey)
373 if create:
374 if self.namespace is not None:
375 if self.namespace not in context._inspector.get_schema_names():
376 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace))
377 # In our tables we have columns that make use of sqlalchemy
378 # Sequence objects. There is currently a bug in sqlalchmey that
379 # causes a deprecation warning to be thrown on a property of
380 # the Sequence object when the repr for the sequence is
381 # created. Here a filter is used to catch these deprecation
382 # warnings when tables are created.
383 with warnings.catch_warnings():
384 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning)
385 self._metadata.create_all(self._connection)
386 except BaseException:
387 self._metadata.drop_all(self._connection)
388 self._metadata = None
389 raise
391 @abstractmethod
392 def isWriteable(self) -> bool:
393 """Return `True` if this database can be modified by this client.
394 """
395 raise NotImplementedError()
397 @abstractmethod
398 def __str__(self) -> str:
399 """Return a human-readable identifier for this `Database`, including
400 any namespace or schema that identifies its names within a `Registry`.
401 """
402 raise NotImplementedError()
404 def shrinkDatabaseEntityName(self, original: str) -> str:
405 """Return a version of the given name that fits within this database
406 engine's length limits for table, constraint, indexes, and sequence
407 names.
409 Implementations should not assume that simple truncation is safe,
410 because multiple long names often begin with the same prefix.
412 The default implementation simply returns the given name.
414 Parameters
415 ----------
416 original : `str`
417 The original name.
419 Returns
420 -------
421 shrunk : `str`
422 The new, possibly shortened name.
423 """
424 return original
426 def expandDatabaseEntityName(self, shrunk: str) -> str:
427 """Retrieve the original name for a database entity that was too long
428 to fit within the database engine's limits.
430 Parameters
431 ----------
432 original : `str`
433 The original name.
435 Returns
436 -------
437 shrunk : `str`
438 The new, possibly shortened name.
439 """
440 return shrunk
442 def _mangleTableName(self, name: str) -> str:
443 """Map a logical, user-visible table name to the true table name used
444 in the database.
446 The default implementation returns the given name unchanged.
448 Parameters
449 ----------
450 name : `str`
451 Input table name. Should not include a namespace (i.e. schema)
452 prefix.
454 Returns
455 -------
456 mangled : `str`
457 Mangled version of the table name (still with no namespace prefix).
459 Notes
460 -----
461 Reimplementations of this method must be idempotent - mangling an
462 already-mangled name must have no effect.
463 """
464 return name
466 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData,
467 **kwds) -> sqlalchemy.schema.Column:
468 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`.
470 Parameters
471 ----------
472 table : `str`
473 Name of the table this column is being added to.
474 spec : `FieldSpec`
475 Specification for the field to be added.
476 metadata : `sqlalchemy.MetaData`
477 SQLAlchemy representation of the DDL schema this field's table is
478 being added to.
479 **kwds
480 Additional keyword arguments to forward to the
481 `sqlalchemy.schema.Column` constructor. This is provided to make
482 it easier for derived classes to delegate to ``super()`` while
483 making only minor changes.
485 Returns
486 -------
487 column : `sqlalchemy.schema.Column`
488 SQLAlchemy representation of the field.
489 """
490 args = [spec.name, spec.getSizedColumnType()]
491 if spec.autoincrement:
492 # Generate a sequence to use for auto incrementing for databases
493 # that do not support it natively. This will be ignored by
494 # sqlalchemy for databases that do support it.
495 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"),
496 metadata=metadata))
497 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}."
498 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey,
499 comment=spec.doc, **kwds)
501 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData,
502 **kwds) -> sqlalchemy.schema.ForeignKeyConstraint:
503 """Convert a `ForeignKeySpec` to a
504 `sqlalchemy.schema.ForeignKeyConstraint`.
506 Parameters
507 ----------
508 table : `str`
509 Name of the table this foreign key is being added to.
510 spec : `ForeignKeySpec`
511 Specification for the foreign key to be added.
512 metadata : `sqlalchemy.MetaData`
513 SQLAlchemy representation of the DDL schema this constraint is
514 being added to.
515 **kwds
516 Additional keyword arguments to forward to the
517 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is
518 provided to make it easier for derived classes to delegate to
519 ``super()`` while making only minor changes.
521 Returns
522 -------
523 constraint : `sqlalchemy.schema.ForeignKeyConstraint`
524 SQLAlchemy representation of the constraint.
525 """
526 name = self.shrinkDatabaseEntityName(
527 "_".join(["fkey", table, self._mangleTableName(spec.table)]
528 + list(spec.target) + list(spec.source))
529 )
530 return sqlalchemy.schema.ForeignKeyConstraint(
531 spec.source,
532 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target],
533 name=name,
534 ondelete=spec.onDelete
535 )
537 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData,
538 **kwds) -> sqlalchemy.schema.Table:
539 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`.
541 Parameters
542 ----------
543 spec : `TableSpec`
544 Specification for the foreign key to be added.
545 metadata : `sqlalchemy.MetaData`
546 SQLAlchemy representation of the DDL schema this table is being
547 added to.
548 **kwds
549 Additional keyword arguments to forward to the
550 `sqlalchemy.schema.Table` constructor. This is provided to make it
551 easier for derived classes to delegate to ``super()`` while making
552 only minor changes.
554 Returns
555 -------
556 table : `sqlalchemy.schema.Table`
557 SQLAlchemy representation of the table.
559 Notes
560 -----
561 This method does not handle ``spec.foreignKeys`` at all, in order to
562 avoid circular dependencies. These are added by higher-level logic in
563 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`.
564 """
565 name = self._mangleTableName(name)
566 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields]
567 args.extend(
568 sqlalchemy.schema.UniqueConstraint(
569 *columns,
570 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns)))
571 )
572 for columns in spec.unique if columns not in spec.indexes
573 )
574 args.extend(
575 sqlalchemy.schema.Index(
576 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))),
577 *columns,
578 unique=(columns in spec.unique)
579 )
580 for columns in spec.indexes
581 )
582 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}."
583 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds)
585 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause:
586 """Ensure that a table with the given name and specification exists,
587 creating it if necessary.
589 Parameters
590 ----------
591 name : `str`
592 Name of the table (not including namespace qualifiers).
593 spec : `TableSpec`
594 Specification for the table. This will be used when creating the
595 table, and *may* be used when obtaining an existing table to check
596 for consistency, but no such check is guaranteed.
598 Returns
599 -------
600 table : `sqlalchemy.schema.Table`
601 SQLAlchemy representation of the table.
603 Raises
604 ------
605 ReadOnlyDatabaseError
606 Raised if `isWriteable` returns `False`, and the table does not
607 already exist.
608 DatabaseConflictError
609 Raised if the table exists but ``spec`` is inconsistent with its
610 definition.
612 Notes
613 -----
614 This method may not be called within transactions. It may be called on
615 read-only databases if and only if the table does in fact already
616 exist.
618 Subclasses may override this method, but usually should not need to.
619 """
620 assert not self._connection.in_transaction(), "Table creation interrupts transactions."
621 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
622 table = self.getExistingTable(name, spec)
623 if table is not None:
624 return table
625 if not self.isWriteable():
626 raise ReadOnlyDatabaseError(
627 f"Table {name} does not exist, and cannot be created "
628 f"because database {self} is read-only."
629 )
630 table = self._convertTableSpec(name, spec, self._metadata)
631 for foreignKeySpec in spec.foreignKeys:
632 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
633 table.create(self._connection)
634 return table
636 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]:
637 """Obtain an existing table with the given name and specification.
639 Parameters
640 ----------
641 name : `str`
642 Name of the table (not including namespace qualifiers).
643 spec : `TableSpec`
644 Specification for the table. This will be used when creating the
645 SQLAlchemy representation of the table, and it is used to
646 check that the actual table in the database is consistent.
648 Returns
649 -------
650 table : `sqlalchemy.schema.Table` or `None`
651 SQLAlchemy representation of the table, or `None` if it does not
652 exist.
654 Raises
655 ------
656 DatabaseConflictError
657 Raised if the table exists but ``spec`` is inconsistent with its
658 definition.
660 Notes
661 -----
662 This method can be called within transactions and never modifies the
663 database.
665 Subclasses may override this method, but usually should not need to.
666 """
667 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
668 name = self._mangleTableName(name)
669 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}")
670 if table is not None:
671 if spec.fields.names != set(table.columns.keys()):
672 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new "
673 f"specification has columns {list(spec.fields.names)}, while "
674 f"the previous definition has {list(table.columns.keys())}.")
675 else:
676 inspector = sqlalchemy.engine.reflection.Inspector(self._connection)
677 if name in inspector.get_table_names(schema=self.namespace):
678 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace))
679 table = self._convertTableSpec(name, spec, self._metadata)
680 for foreignKeySpec in spec.foreignKeys:
681 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
682 return table
683 return table
685 def sync(self, table: sqlalchemy.schema.Table, *,
686 keys: Dict[str, Any],
687 compared: Optional[Dict[str, Any]] = None,
688 extra: Optional[Dict[str, Any]] = None,
689 returning: Optional[Sequence[str]] = None,
690 ) -> Tuple[Optional[Dict[str, Any], bool]]:
691 """Insert into a table as necessary to ensure database contains
692 values equivalent to the given ones.
694 Parameters
695 ----------
696 table : `sqlalchemy.schema.Table`
697 Table to be queried and possibly inserted into.
698 keys : `dict`
699 Column name-value pairs used to search for an existing row; must
700 be a combination that can be used to select a single row if one
701 exists. If such a row does not exist, these values are used in
702 the insert.
703 compared : `dict`, optional
704 Column name-value pairs that are compared to those in any existing
705 row. If such a row does not exist, these rows are used in the
706 insert.
707 extra : `dict`, optional
708 Column name-value pairs that are ignored if a matching row exists,
709 but used in an insert if one is necessary.
710 returning : `~collections.abc.Sequence` of `str`, optional
711 The names of columns whose values should be returned.
713 Returns
714 -------
715 row : `dict`, optional
716 The value of the fields indicated by ``returning``, or `None` if
717 ``returning`` is `None`.
718 inserted : `bool`
719 If `True`, a new row was inserted.
721 Raises
722 ------
723 DatabaseConflictError
724 Raised if the values in ``compared`` do not match the values in the
725 database.
726 ReadOnlyDatabaseError
727 Raised if `isWriteable` returns `False`, and no matching record
728 already exists.
730 Notes
731 -----
732 This method may not be called within transactions. It may be called on
733 read-only databases if and only if the matching row does in fact
734 already exist.
735 """
737 def check():
738 """Query for a row that matches the ``key`` argument, and compare
739 to what was given by the caller.
741 Returns
742 -------
743 n : `int`
744 Number of matching rows. ``n != 1`` is always an error, but
745 it's a different kind of error depending on where `check` is
746 being called.
747 bad : `list` of `str`, or `None`
748 The subset of the keys of ``compared`` for which the existing
749 values did not match the given one. Once again, ``not bad``
750 is always an error, but a different kind on context. `None`
751 if ``n != 1``
752 result : `list` or `None`
753 Results in the database that correspond to the columns given
754 in ``returning``, or `None` if ``returning is None``.
755 """
756 toSelect = set()
757 if compared is not None:
758 toSelect.update(compared.keys())
759 if returning is not None:
760 toSelect.update(returning)
761 if not toSelect:
762 # Need to select some column, even if we just want to see
763 # how many rows we get back.
764 toSelect.add(next(iter(keys.keys())))
765 selectSql = sqlalchemy.sql.select(
766 [table.columns[k].label(k) for k in toSelect]
767 ).select_from(table).where(
768 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])
769 )
770 fetched = list(self._connection.execute(selectSql).fetchall())
771 if len(fetched) != 1:
772 return len(fetched), None, None
773 existing = fetched[0]
774 if compared is not None:
775 inconsistencies = [k for k, v in compared.items() if existing[k] != v]
776 else:
777 inconsistencies = []
778 if returning is not None:
779 toReturn = [existing[k] for k in returning]
780 else:
781 toReturn = None
782 return 1, inconsistencies, toReturn
784 if self.isWriteable():
785 # Database is writeable. Try an insert first, but allow it to fail
786 # (in only specific ways).
787 row = keys.copy()
788 if compared is not None:
789 row.update(compared)
790 if extra is not None:
791 row.update(extra)
792 insertSql = table.insert().values(row)
793 try:
794 with self.transaction(interrupting=True):
795 self._connection.execute(insertSql)
796 # Need to perform check() for this branch inside the
797 # transaction, so we roll back an insert that didn't do
798 # what we expected. That limits the extent to which we
799 # can reduce duplication between this block and the other
800 # ones that perform similar logic.
801 n, bad, result = check()
802 if n < 1:
803 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.")
804 elif n > 2:
805 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
806 f"unique constraint for table {table.name}.")
807 elif bad:
808 raise RuntimeError("Conflict in sync after successful insert; this should only be "
809 "possible if the same table is being updated by a concurrent "
810 "process that isn't using sync.")
811 # No exceptions, so it looks like we inserted the requested row
812 # successfully.
813 inserted = True
814 except sqlalchemy.exc.IntegrityError as err:
815 # Most likely cause is that an equivalent row already exists,
816 # but it could also be some other constraint. Query for the
817 # row we think we matched to resolve that question.
818 n, bad, result = check()
819 if n < 1:
820 # There was no matched row; insertion failed for some
821 # completely different reason. Just re-raise the original
822 # IntegrityError.
823 raise
824 elif n > 2:
825 # There were multiple matched rows, which means we
826 # conflicted *and* the arguments were bad to begin with.
827 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
828 f"unique constraint for table {table.name}.") from err
829 elif bad:
830 # No logic bug, but data conflicted on the keys given.
831 raise DatabaseConflictError(f"Conflict in sync for table "
832 f"{table.name} on column(s) {bad}.") from err
833 # The desired row is already present and consistent with what
834 # we tried to insert.
835 inserted = False
836 else:
837 assert not self._connection.in_transaction(), (
838 "Calling sync within a transaction block is an error even "
839 "on a read-only database."
840 )
841 # Database is not writeable; just see if the row exists.
842 n, bad, result = check()
843 if n < 1:
844 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.")
845 elif n > 1:
846 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.")
847 elif bad:
848 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.")
849 inserted = False
850 return {k: v for k, v in zip(returning, result)} if returning is not None else None, inserted
852 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False,
853 ) -> Optional[List[int]]:
854 """Insert one or more rows into a table, optionally returning
855 autoincrement primary key values.
857 Parameters
858 ----------
859 table : `sqlalchemy.schema.Table`
860 Table rows should be inserted into.
861 returnIds: `bool`
862 If `True` (`False` is default), return the values of the table's
863 autoincrement primary key field (which much exist).
864 *rows
865 Positional arguments are the rows to be inserted, as dictionaries
866 mapping column name to value. The keys in all dictionaries must
867 be the same.
869 Returns
870 -------
871 ids : `None`, or `list` of `int`
872 If ``returnIds`` is `True`, a `list` containing the inserted
873 values for the table's autoincrement primary key.
875 Raises
876 ------
877 ReadOnlyDatabaseError
878 Raised if `isWriteable` returns `False` when this method is called.
880 Notes
881 -----
882 The default implementation uses bulk insert syntax when ``returnIds``
883 is `False`, and a loop over single-row insert operations when it is
884 `True`.
886 Derived classes should reimplement when they can provide a more
887 efficient implementation (especially for the latter case).
889 May be used inside transaction contexts, so implementations may not
890 perform operations that interrupt transactions.
891 """
892 if not self.isWriteable():
893 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.")
894 if not returnIds:
895 self._connection.execute(table.insert(), *rows)
896 else:
897 sql = table.insert()
898 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows]
900 @abstractmethod
901 def replace(self, table: sqlalchemy.schema.Table, *rows: dict):
902 """Insert one or more rows into a table, replacing any existing rows
903 for which insertion of a new row would violate the primary key
904 constraint.
906 Parameters
907 ----------
908 table : `sqlalchemy.schema.Table`
909 Table rows should be inserted into.
910 *rows
911 Positional arguments are the rows to be inserted, as dictionaries
912 mapping column name to value. The keys in all dictionaries must
913 be the same.
915 Raises
916 ------
917 ReadOnlyDatabaseError
918 Raised if `isWriteable` returns `False` when this method is called.
920 Notes
921 -----
922 May be used inside transaction contexts, so implementations may not
923 perform operations that interrupt transactions.
925 Implementations should raise a `sqlalchemy.exc.IntegrityError`
926 exception when a constraint other than the primary key would be
927 violated.
929 Implementations are not required to support `replace` on tables
930 with autoincrement keys.
931 """
932 raise NotImplementedError()
934 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int:
935 """Delete one or more rows from a table.
937 Parameters
938 ----------
939 table : `sqlalchemy.schema.Table`
940 Table that rows should be deleted from.
941 columns: `~collections.abc.Iterable` of `str`
942 The names of columns that will be used to constrain the rows to
943 be deleted; these will be combined via ``AND`` to form the
944 ``WHERE`` clause of the delete query.
945 *rows
946 Positional arguments are the keys of rows to be deleted, as
947 dictionaries mapping column name to value. The keys in all
948 dictionaries must exactly the names in ``columns``.
950 Returns
951 -------
952 count : `int`
953 Number of rows deleted.
955 Raises
956 ------
957 ReadOnlyDatabaseError
958 Raised if `isWriteable` returns `False` when this method is called.
960 Notes
961 -----
962 May be used inside transaction contexts, so implementations may not
963 perform operations that interrupt transactions.
965 The default implementation should be sufficient for most derived
966 classes.
967 """
968 if not self.isWriteable():
969 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.")
970 sql = table.delete()
971 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns]
972 if whereTerms:
973 sql = sql.where(sqlalchemy.sql.and_(*whereTerms))
974 return self._connection.execute(sql, *rows).rowcount
976 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int:
977 """Update one or more rows in a table.
979 Parameters
980 ----------
981 table : `sqlalchemy.schema.Table`
982 Table containing the rows to be updated.
983 where : `dict` [`str`, `str`]
984 A mapping from the names of columns that will be used to search for
985 existing rows to the keys that will hold these values in the
986 ``rows`` dictionaries. Note that these may not be the same due to
987 SQLAlchemy limitations.
988 *rows
989 Positional arguments are the rows to be updated. The keys in all
990 dictionaries must be the same, and may correspond to either a
991 value in the ``where`` dictionary or the name of a column to be
992 updated.
994 Returns
995 -------
996 count : `int`
997 Number of rows matched (regardless of whether the update actually
998 modified them).
1000 Raises
1001 ------
1002 ReadOnlyDatabaseError
1003 Raised if `isWriteable` returns `False` when this method is called.
1005 Notes
1006 -----
1007 May be used inside transaction contexts, so implementations may not
1008 perform operations that interrupt transactions.
1010 The default implementation should be sufficient for most derived
1011 classes.
1012 """
1013 if not self.isWriteable():
1014 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.")
1015 sql = table.update().where(
1016 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()])
1017 )
1018 return self._connection.execute(sql, *rows).rowcount
1020 def query(self, sql: sqlalchemy.sql.FromClause, *args, **kwds) -> sqlalchemy.engine.ResultProxy:
1021 """Run a SELECT query against the database.
1023 Parameters
1024 ----------
1025 sql : `sqlalchemy.sql.FromClause`
1026 A SQLAlchemy representation of a ``SELECT`` query.
1027 *args
1028 Additional positional arguments are forwarded to
1029 `sqlalchemy.engine.Connection.execute`.
1030 **kwds
1031 Additional keyword arguments are forwarded to
1032 `sqlalchemy.engine.Connection.execute`.
1034 Returns
1035 -------
1036 result : `sqlalchemy.engine.ResultProxy`
1037 Query results.
1039 Notes
1040 -----
1041 The default implementation should be sufficient for most derived
1042 classes.
1043 """
1044 # TODO: should we guard against non-SELECT queries here?
1045 return self._connection.execute(sql, *args, **kwds)
1047 origin: int
1048 """An integer ID that should be used as the default for any datasets,
1049 quanta, or other entities that use a (autoincrement, origin) compound
1050 primary key (`int`).
1051 """
1053 namespace: Optional[str]
1054 """The schema or namespace this database instance is associated with
1055 (`str` or `None`).
1056 """