Coverage for python/lsst/daf/butler/registry/interfaces/_database.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = [
24 "Database",
25 "ReadOnlyDatabaseError",
26 "DatabaseConflictError",
27 "StaticTablesContext",
28]
30from abc import ABC, abstractmethod
31from contextlib import contextmanager
32from typing import (
33 Any,
34 Dict,
35 Iterable,
36 List,
37 Optional,
38 Sequence,
39 Tuple,
40)
41import warnings
43import sqlalchemy
45from ...core import ddl
48def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: Dict[str, Any]):
49 """Test that the definition of a table in a `ddl.TableSpec` and from
50 database introspection are consistent.
52 Parameters
53 ----------
54 name : `str`
55 Name of the table (only used in error messages).
56 spec : `ddl.TableSpec`
57 Specification of the table.
58 inspection : `dict`
59 Dictionary returned by
60 `sqlalchemy.engine.reflection.Inspector.get_columns`.
62 Raises
63 ------
64 DatabaseConflictError
65 Raised if the definitions are inconsistent.
66 """
67 columnNames = [c["name"] for c in inspection]
68 if spec.fields.names != set(columnNames):
69 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; "
70 f"specification has columns {list(spec.fields.names)}, while the "
71 f"table in the database has {columnNames}.")
74class ReadOnlyDatabaseError(RuntimeError):
75 """Exception raised when a write operation is called on a read-only
76 `Database`.
77 """
80class DatabaseConflictError(RuntimeError):
81 """Exception raised when database content (row values or schema entities)
82 are inconsistent with what this client expects.
83 """
86class StaticTablesContext:
87 """Helper class used to declare the static schema for a registry layer
88 in a database.
90 An instance of this class is returned by `Database.declareStaticTables`,
91 which should be the only way it should be constructed.
92 """
94 def __init__(self, db: Database):
95 self._db = db
96 self._foreignKeys = []
97 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection)
98 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace))
100 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
101 """Add a new table to the schema, returning its sqlalchemy
102 representation.
104 The new table may not actually be created until the end of the
105 context created by `Database.declareStaticTables`, allowing tables
106 to be declared in any order even in the presence of foreign key
107 relationships.
108 """
109 if name in self._tableNames:
110 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name,
111 schema=self._db.namespace))
112 table = self._db._convertTableSpec(name, spec, self._db._metadata)
113 for foreignKeySpec in spec.foreignKeys:
114 self._foreignKeys.append(
115 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata))
116 )
117 return table
119 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]:
120 """Add a named tuple of tables to the schema, returning their
121 SQLAlchemy representations in a named tuple of the same type.
123 The new tables may not actually be created until the end of the
124 context created by `Database.declareStaticTables`, allowing tables
125 to be declared in any order even in the presence of foreign key
126 relationships.
128 Notes
129 -----
130 ``specs`` *must* be an instance of a type created by
131 `collections.namedtuple`, not just regular tuple, and the returned
132 object is guaranteed to be the same. Because `~collections.namedtuple`
133 is just a factory for `type` objects, not an actual type itself,
134 we cannot represent this with type annotations.
135 """
136 return specs._make(self.addTable(name, spec) for name, spec in zip(specs._fields, specs))
139class Database(ABC):
140 """An abstract interface that represents a particular database engine's
141 representation of a single schema/namespace/database.
143 Parameters
144 ----------
145 origin : `int`
146 An integer ID that should be used as the default for any datasets,
147 quanta, or other entities that use a (autoincrement, origin) compound
148 primary key.
149 connection : `sqlalchemy.engine.Connection`
150 The SQLAlchemy connection this `Database` wraps.
151 namespace : `str`, optional
152 Name of the schema or namespace this instance is associated with.
153 This is passed as the ``schema`` argument when constructing a
154 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to
155 avoid confusion between "schema means namespace" and "schema means
156 table definitions".
158 Notes
159 -----
160 `Database` requires all write operations to go through its special named
161 methods. Our write patterns are sufficiently simple that we don't really
162 need the full flexibility of SQL insert/update/delete syntax, and we need
163 non-standard (but common) functionality in these operations sufficiently
164 often that it seems worthwhile to provide our own generic API.
166 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via
167 their SQLAlchemy representation) to be run, as we expect these to require
168 significantly more sophistication while still being limited to standard
169 SQL.
171 `Database` itself has several underscore-prefixed attributes:
173 - ``_cs``: SQLAlchemy objects representing the connection and transaction
174 state.
175 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing
176 the tables and other schema entities.
178 These are considered protected (derived classes may access them, but other
179 code should not), and read-only, aside from executing SQL via
180 ``_connection``.
181 """
183 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection,
184 namespace: Optional[str] = None):
185 self.origin = origin
186 self.namespace = namespace
187 self._connection = connection
188 self._metadata = None
190 @classmethod
191 def makeDefaultUri(cls, root: str) -> Optional[str]:
192 """Create a default connection URI appropriate for the given root
193 directory, or `None` if there can be no such default.
194 """
195 return None
197 @classmethod
198 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None,
199 writeable: bool = True) -> Database:
200 """Construct a database from a SQLAlchemy URI.
202 Parameters
203 ----------
204 uri : `str`
205 A SQLAlchemy URI connection string.
206 origin : `int`
207 An integer ID that should be used as the default for any datasets,
208 quanta, or other entities that use a (autoincrement, origin)
209 compound primary key.
210 namespace : `str`, optional
211 A database namespace (i.e. schema) the new instance should be
212 associated with. If `None` (default), the namespace (if any) is
213 inferred from the URI.
214 writeable : `bool`, optional
215 If `True`, allow write operations on the database, including
216 ``CREATE TABLE``.
218 Returns
219 -------
220 db : `Database`
221 A new `Database` instance.
222 """
223 return cls.fromConnection(cls.connect(uri, writeable=writeable),
224 origin=origin,
225 namespace=namespace,
226 writeable=writeable)
228 @classmethod
229 @abstractmethod
230 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection:
231 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI.
233 Parameters
234 ----------
235 uri : `str`
236 A SQLAlchemy URI connection string.
237 origin : `int`
238 An integer ID that should be used as the default for any datasets,
239 quanta, or other entities that use a (autoincrement, origin)
240 compound primary key.
241 writeable : `bool`, optional
242 If `True`, allow write operations on the database, including
243 ``CREATE TABLE``.
245 Returns
246 -------
247 connection : `sqlalchemy.engine.Connection`
248 A database connection.
250 Notes
251 -----
252 Subclasses that support other ways to connect to a database are
253 encouraged to add optional arguments to their implementation of this
254 method, as long as they maintain compatibility with the base class
255 call signature.
256 """
257 raise NotImplementedError()
259 @classmethod
260 @abstractmethod
261 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int,
262 namespace: Optional[str] = None, writeable: bool = True) -> Database:
263 """Create a new `Database` from an existing
264 `sqlalchemy.engine.Connection`.
266 Parameters
267 ----------
268 connection : `sqllachemy.engine.Connection`
269 The connection for the the database. May be shared between
270 `Database` instances.
271 origin : `int`
272 An integer ID that should be used as the default for any datasets,
273 quanta, or other entities that use a (autoincrement, origin)
274 compound primary key.
275 namespace : `str`, optional
276 A different database namespace (i.e. schema) the new instance
277 should be associated with. If `None` (default), the namespace
278 (if any) is inferred from the connection.
279 writeable : `bool`, optional
280 If `True`, allow write operations on the database, including
281 ``CREATE TABLE``.
283 Returns
284 -------
285 db : `Database`
286 A new `Database` instance.
288 Notes
289 -----
290 This method allows different `Database` instances to share the same
291 connection, which is desirable when they represent different namespaces
292 can be queried together. This also ties their transaction state,
293 however; starting a transaction in any database automatically starts
294 on in all other databases.
295 """
296 raise NotImplementedError()
298 @contextmanager
299 def transaction(self, *, interrupting: bool = False) -> None:
300 """Return a context manager that represents a transaction.
302 Parameters
303 ----------
304 interrupting : `bool`
305 If `True`, this transaction block needs to be able to interrupt
306 any existing one in order to yield correct behavior.
307 """
308 assert not (interrupting and self._connection.in_transaction()), (
309 "Logic error in transaction nesting: an operation that would "
310 "interrupt the active transaction context has been requested."
311 )
312 if self._connection.in_transaction():
313 trans = self._connection.begin_nested()
314 else:
315 # Use a regular (non-savepoint) transaction only for the outermost
316 # context.
317 trans = self._connection.begin()
318 try:
319 yield
320 trans.commit()
321 except BaseException:
322 trans.rollback()
323 raise
325 @contextmanager
326 def declareStaticTables(self, *, create: bool) -> StaticTablesContext:
327 """Return a context manager in which the database's static DDL schema
328 can be declared.
330 Parameters
331 ----------
332 create : `bool`
333 If `True`, attempt to create all tables at the end of the context.
334 If `False`, they will be assumed to already exist.
336 Returns
337 -------
338 schema : `StaticTablesContext`
339 A helper object that is used to add new tables.
341 Raises
342 ------
343 ReadOnlyDatabaseError
344 Raised if ``create`` is `True`, `Database.isWriteable` is `False`,
345 and one or more declared tables do not already exist.
347 Examples
348 --------
349 Given a `Database` instance ``db``::
351 with db.declareStaticTables(create=True) as schema:
352 schema.addTable("table1", TableSpec(...))
353 schema.addTable("table2", TableSpec(...))
355 Notes
356 -----
357 A database's static DDL schema must be declared before any dynamic
358 tables are managed via calls to `ensureTableExists` or
359 `getExistingTable`. The order in which static schema tables are added
360 inside the context block is unimportant; they will automatically be
361 sorted and added in an order consistent with their foreign key
362 relationships.
363 """
364 if create and not self.isWriteable():
365 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.")
366 self._metadata = sqlalchemy.MetaData(schema=self.namespace)
367 try:
368 context = StaticTablesContext(self)
369 yield context
370 for table, foreignKey in context._foreignKeys:
371 table.append_constraint(foreignKey)
372 if create:
373 if self.namespace is not None:
374 if self.namespace not in context._inspector.get_schema_names():
375 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace))
376 # In our tables we have columns that make use of sqlalchemy
377 # Sequence objects. There is currently a bug in sqlalchmey that
378 # causes a deprecation warning to be thrown on a property of
379 # the Sequence object when the repr for the sequence is
380 # created. Here a filter is used to catch these deprecation
381 # warnings when tables are created.
382 with warnings.catch_warnings():
383 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning)
384 self._metadata.create_all(self._connection)
385 except BaseException:
386 self._metadata.drop_all(self._connection)
387 self._metadata = None
388 raise
390 @abstractmethod
391 def isWriteable(self) -> bool:
392 """Return `True` if this database can be modified by this client.
393 """
394 raise NotImplementedError()
396 @abstractmethod
397 def __str__(self) -> str:
398 """Return a human-readable identifier for this `Database`, including
399 any namespace or schema that identifies its names within a `Registry`.
400 """
401 raise NotImplementedError()
403 def shrinkDatabaseEntityName(self, original: str) -> str:
404 """Return a version of the given name that fits within this database
405 engine's length limits for table, constraint, indexes, and sequence
406 names.
408 Implementations should not assume that simple truncation is safe,
409 because multiple long names often begin with the same prefix.
411 The default implementation simply returns the given name.
413 Parameters
414 ----------
415 original : `str`
416 The original name.
418 Returns
419 -------
420 shrunk : `str`
421 The new, possibly shortened name.
422 """
423 return original
425 def expandDatabaseEntityName(self, shrunk: str) -> str:
426 """Retrieve the original name for a database entity that was too long
427 to fit within the database engine's limits.
429 Parameters
430 ----------
431 original : `str`
432 The original name.
434 Returns
435 -------
436 shrunk : `str`
437 The new, possibly shortened name.
438 """
439 return shrunk
441 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData,
442 **kwds) -> sqlalchemy.schema.Column:
443 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`.
445 Parameters
446 ----------
447 table : `str`
448 Name of the table this column is being added to.
449 spec : `FieldSpec`
450 Specification for the field to be added.
451 metadata : `sqlalchemy.MetaData`
452 SQLAlchemy representation of the DDL schema this field's table is
453 being added to.
454 **kwds
455 Additional keyword arguments to forward to the
456 `sqlalchemy.schema.Column` constructor. This is provided to make
457 it easier for derived classes to delegate to ``super()`` while
458 making only minor changes.
460 Returns
461 -------
462 column : `sqlalchemy.schema.Column`
463 SQLAlchemy representation of the field.
464 """
465 args = [spec.name, spec.getSizedColumnType()]
466 if spec.autoincrement:
467 # Generate a sequence to use for auto incrementing for databases
468 # that do not support it natively. This will be ignored by
469 # sqlalchemy for databases that do support it.
470 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"),
471 metadata=metadata))
472 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}."
473 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey,
474 comment=spec.doc, **kwds)
476 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData,
477 **kwds) -> sqlalchemy.schema.ForeignKeyConstraint:
478 """Convert a `ForeignKeySpec` to a
479 `sqlalchemy.schema.ForeignKeyConstraint`.
481 Parameters
482 ----------
483 table : `str`
484 Name of the table this foreign key is being added to.
485 spec : `ForeignKeySpec`
486 Specification for the foreign key to be added.
487 metadata : `sqlalchemy.MetaData`
488 SQLAlchemy representation of the DDL schema this constraint is
489 being added to.
490 **kwds
491 Additional keyword arguments to forward to the
492 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is
493 provided to make it easier for derived classes to delegate to
494 ``super()`` while making only minor changes.
496 Returns
497 -------
498 constraint : `sqlalchemy.schema.ForeignKeyConstraint`
499 SQLAlchemy representation of the constraint.
500 """
501 name = self.shrinkDatabaseEntityName(
502 "_".join(["fkey", table, spec.table] + list(spec.target) + list(spec.source))
503 )
504 return sqlalchemy.schema.ForeignKeyConstraint(
505 spec.source,
506 [f"{spec.table}.{col}" for col in spec.target],
507 name=name,
508 ondelete=spec.onDelete
509 )
511 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData,
512 **kwds) -> sqlalchemy.schema.Table:
513 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`.
515 Parameters
516 ----------
517 spec : `TableSpec`
518 Specification for the foreign key to be added.
519 metadata : `sqlalchemy.MetaData`
520 SQLAlchemy representation of the DDL schema this table is being
521 added to.
522 **kwds
523 Additional keyword arguments to forward to the
524 `sqlalchemy.schema.Table` constructor. This is provided to make it
525 easier for derived classes to delegate to ``super()`` while making
526 only minor changes.
528 Returns
529 -------
530 table : `sqlalchemy.schema.Table`
531 SQLAlchemy representation of the table.
533 Notes
534 -----
535 This method does not handle ``spec.foreignKeys`` at all, in order to
536 avoid circular dependencies. These are added by higher-level logic in
537 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`.
538 """
539 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields]
540 args.extend(
541 sqlalchemy.schema.UniqueConstraint(
542 *columns,
543 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns)))
544 )
545 for columns in spec.unique if columns not in spec.indexes
546 )
547 args.extend(
548 sqlalchemy.schema.Index(
549 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))),
550 *columns,
551 unique=(columns in spec.unique)
552 )
553 for columns in spec.indexes
554 )
555 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}."
556 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds)
558 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause:
559 """Ensure that a table with the given name and specification exists,
560 creating it if necessary.
562 Parameters
563 ----------
564 name : `str`
565 Name of the table (not including namespace qualifiers).
566 spec : `TableSpec`
567 Specification for the table. This will be used when creating the
568 table, and *may* be used when obtaining an existing table to check
569 for consistency, but no such check is guaranteed.
571 Returns
572 -------
573 table : `sqlalchemy.schema.Table`
574 SQLAlchemy representation of the table.
576 Raises
577 ------
578 ReadOnlyDatabaseError
579 Raised if `isWriteable` returns `False`, and the table does not
580 already exist.
581 DatabaseConflictError
582 Raised if the table exists but ``spec`` is inconsistent with its
583 definition.
585 Notes
586 -----
587 This method may not be called within transactions. It may be called on
588 read-only databases if and only if the table does in fact already
589 exist.
591 Subclasses may override this method, but usually should not need to.
592 """
593 assert not self._connection.in_transaction(), "Table creation interrupts transactions."
594 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
595 table = self.getExistingTable(name, spec)
596 if table is not None:
597 return table
598 if not self.isWriteable():
599 raise ReadOnlyDatabaseError(
600 f"Table {name} does not exist, and cannot be created "
601 f"because database {self} is read-only."
602 )
603 table = self._convertTableSpec(name, spec, self._metadata)
604 for foreignKeySpec in spec.foreignKeys:
605 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
606 table.create(self._connection)
607 return table
609 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]:
610 """Obtain an existing table with the given name and specification.
612 Parameters
613 ----------
614 name : `str`
615 Name of the table (not including namespace qualifiers).
616 spec : `TableSpec`
617 Specification for the table. This will be used when creating the
618 SQLAlchemy representation of the table, and it is used to
619 check that the actual table in the database is consistent.
621 Returns
622 -------
623 table : `sqlalchemy.schema.Table` or `None`
624 SQLAlchemy representation of the table, or `None` if it does not
625 exist.
627 Raises
628 ------
629 DatabaseConflictError
630 Raised if the table exists but ``spec`` is inconsistent with its
631 definition.
633 Notes
634 -----
635 This method can be called within transactions and never modifies the
636 database.
638 Subclasses may override this method, but usually should not need to.
639 """
640 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
641 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}")
642 if table is not None:
643 if spec.fields.names != set(table.columns.keys()):
644 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new "
645 f"specification has columns {list(spec.fields.names)}, while "
646 f"the previous definition has {list(table.columns.keys())}.")
647 else:
648 inspector = sqlalchemy.engine.reflection.Inspector(self._connection)
649 if name in inspector.get_table_names(schema=self.namespace):
650 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace))
651 table = self._convertTableSpec(name, spec, self._metadata)
652 for foreignKeySpec in spec.foreignKeys:
653 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
654 return table
655 return table
657 def sync(self, table: sqlalchemy.schema.Table, *,
658 keys: Dict[str, Any],
659 compared: Optional[Dict[str, Any]] = None,
660 extra: Optional[Dict[str, Any]] = None,
661 returning: Optional[Sequence[str]] = None,
662 ) -> Tuple[Optional[Dict[str, Any], bool]]:
663 """Insert into a table as necessary to ensure database contains
664 values equivalent to the given ones.
666 Parameters
667 ----------
668 table : `sqlalchemy.schema.Table`
669 Table to be queried and possibly inserted into.
670 keys : `dict`
671 Column name-value pairs used to search for an existing row; must
672 be a combination that can be used to select a single row if one
673 exists. If such a row does not exist, these values are used in
674 the insert.
675 compared : `dict`, optional
676 Column name-value pairs that are compared to those in any existing
677 row. If such a row does not exist, these rows are used in the
678 insert.
679 extra : `dict`, optional
680 Column name-value pairs that are ignored if a matching row exists,
681 but used in an insert if one is necessary.
682 returning : `~collections.abc.Sequence` of `str`, optional
683 The names of columns whose values should be returned.
685 Returns
686 -------
687 row : `dict`, optional
688 The value of the fields indicated by ``returning``, or `None` if
689 ``returning`` is `None`.
690 inserted : `bool`
691 If `True`, a new row was inserted.
693 Raises
694 ------
695 DatabaseConflictErrorError
696 Raised if the values in ``compared`` do match the values in the
697 database.
698 ReadOnlyDatabaseError
699 Raised if `isWriteable` returns `False`, and no matching record
700 already exists.
702 Notes
703 -----
704 This method may not be called within transactions. It may be called on
705 read-only databases if and only if the matching row does in fact
706 already exist.
707 """
709 def check():
710 """Query for a row that matches the ``key`` argument, and compare
711 to what was given by the caller.
713 Returns
714 -------
715 n : `int`
716 Number of matching rows. ``n != 1`` is always an error, but
717 it's a different kind of error depending on where `check` is
718 being called.
719 bad : `list` of `str`, or `None`
720 The subset of the keys of ``compared`` for which the existing
721 values did not match the given one. Once again, ``not bad``
722 is always an error, but a different kind on context. `None`
723 if ``n != 1``
724 result : `list` or `None`
725 Results in the database that correspond to the columns given
726 in ``returning``, or `None` if ``returning is None``.
727 """
728 toSelect = set()
729 if compared is not None:
730 toSelect.update(compared.keys())
731 if returning is not None:
732 toSelect.update(returning)
733 if not toSelect:
734 # Need to select some column, even if we just want to see
735 # how many rows we get back.
736 toSelect.add(next(iter(keys.keys())))
737 selectSql = sqlalchemy.sql.select(
738 [table.columns[k].label(k) for k in toSelect]
739 ).select_from(table).where(
740 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])
741 )
742 fetched = list(self._connection.execute(selectSql).fetchall())
743 if len(fetched) != 1:
744 return len(fetched), None, None
745 existing = fetched[0]
746 if compared is not None:
747 inconsistencies = [k for k, v in compared.items() if existing[k] != v]
748 else:
749 inconsistencies = []
750 if returning is not None:
751 toReturn = [existing[k] for k in returning]
752 else:
753 toReturn = None
754 return 1, inconsistencies, toReturn
756 if self.isWriteable():
757 # Database is writeable. Try an insert first, but allow it to fail
758 # (in only specific ways).
759 row = keys.copy()
760 if compared is not None:
761 row.update(compared)
762 if extra is not None:
763 row.update(extra)
764 insertSql = table.insert().values(row)
765 try:
766 with self.transaction(interrupting=True):
767 self._connection.execute(insertSql)
768 # Need to perform check() for this branch inside the
769 # transaction, so we roll back an insert that didn't do
770 # what we expected. That limits the extent to which we
771 # can reduce duplication between this block and the other
772 # ones that perform similar logic.
773 n, bad, result = check()
774 if n < 1:
775 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.")
776 elif n > 2:
777 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
778 f"unique constraint for table {table.name}.")
779 elif bad:
780 raise RuntimeError("Conflict in sync after successful insert; this should only be "
781 "possible if the same table is being updated by a concurrent "
782 "process that isn't using sync.")
783 # No exceptions, so it looks like we inserted the requested row
784 # successfully.
785 inserted = True
786 except sqlalchemy.exc.IntegrityError as err:
787 # Most likely cause is that an equivalent row already exists,
788 # but it could also be some other constraint. Query for the
789 # row we think we matched to resolve that question.
790 n, bad, result = check()
791 if n < 1:
792 # There was no matched row; insertion failed for some
793 # completely different reason. Just re-raise the original
794 # IntegrityError.
795 raise
796 elif n > 2:
797 # There were multiple matched rows, which means we
798 # conflicted *and* the arguments were bad to begin with.
799 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
800 f"unique constraint for table {table.name}.") from err
801 elif bad:
802 # No logic bug, but data conflicted on the keys given.
803 raise DatabaseConflictError(f"Conflict in sync for table "
804 f"{table.name} on column(s) {bad}.") from err
805 # The desired row is already present and consistent with what
806 # we tried to insert.
807 inserted = False
808 else:
809 assert not self._connection.in_transaction(), (
810 "Calling sync within a transaction block is an error even "
811 "on a read-only database."
812 )
813 # Database is not writeable; just see if the row exists.
814 n, bad, result = check()
815 if n < 1:
816 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.")
817 elif n > 1:
818 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.")
819 elif bad:
820 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.")
821 inserted = False
822 return result, inserted
824 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False,
825 ) -> Optional[List[int]]:
826 """Insert one or more rows into a table, optionally returning
827 autoincrement primary key values.
829 Parameters
830 ----------
831 table : `sqlalchemy.schema.Table`
832 Table rows should be inserted into.
833 returnIds: `bool`
834 If `True` (`False` is default), return the values of the table's
835 autoincrement primary key field (which much exist).
836 *rows
837 Positional arguments are the rows to be inserted, as dictionaries
838 mapping column name to value. The keys in all dictionaries must
839 be the same.
841 Returns
842 -------
843 ids : `None`, or `list` of `int`
844 If ``returnIds`` is `True`, a `list` containing the inserted
845 values for the table's autoincrement primary key.
847 Raises
848 ------
849 ReadOnlyDatabaseError
850 Raised if `isWriteable` returns `False` when this method is called.
852 Notes
853 -----
854 The default implementation uses bulk insert syntax when ``returnIds``
855 is `False`, and a loop over single-row insert operations when it is
856 `True`.
858 Derived classes should reimplement when they can provide a more
859 efficient implementation (especially for the latter case).
861 May be used inside transaction contexts, so implementations may not
862 perform operations that interrupt transactions.
863 """
864 if not self.isWriteable():
865 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.")
866 if not returnIds:
867 self._connection.execute(table.insert(), *rows)
868 else:
869 sql = table.insert()
870 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows]
872 @abstractmethod
873 def replace(self, table: sqlalchemy.schema.Table, *rows: dict):
874 """Insert one or more rows into a table, replacing any existing rows
875 for which insertion of a new row would violate the primary key
876 constraint.
878 Parameters
879 ----------
880 table : `sqlalchemy.schema.Table`
881 Table rows should be inserted into.
882 *rows
883 Positional arguments are the rows to be inserted, as dictionaries
884 mapping column name to value. The keys in all dictionaries must
885 be the same.
887 Raises
888 ------
889 ReadOnlyDatabaseError
890 Raised if `isWriteable` returns `False` when this method is called.
892 Notes
893 -----
894 May be used inside transaction contexts, so implementations may not
895 perform operations that interrupt transactions.
897 Implementations should raise a `sqlalchemy.exc.IntegrityError`
898 exception when a constraint other than the primary key would be
899 violated.
901 Implementations are not required to support `replace` on tables
902 with autoincrement keys.
903 """
904 raise NotImplementedError()
906 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int:
907 """Delete one or more rows from a table.
909 Parameters
910 ----------
911 table : `sqlalchemy.schema.Table`
912 Table that rows should be deleted from.
913 columns: `~collections.abc.Iterable` of `str`
914 The names of columns that will be used to constrain the rows to
915 be deleted; these will be combined via ``AND`` to form the
916 ``WHERE`` clause of the delete query.
917 *rows
918 Positional arguments are the keys of rows to be deleted, as
919 dictionaries mapping column name to value. The keys in all
920 dictionaries must exactly the names in ``columns``.
922 Returns
923 -------
924 count : `int`
925 Number of rows deleted.
927 Raises
928 ------
929 ReadOnlyDatabaseError
930 Raised if `isWriteable` returns `False` when this method is called.
932 Notes
933 -----
934 May be used inside transaction contexts, so implementations may not
935 perform operations that interrupt transactions.
937 The default implementation should be sufficient for most derived
938 classes.
939 """
940 if not self.isWriteable():
941 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.")
942 sql = table.delete()
943 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns]
944 if whereTerms:
945 sql = sql.where(sqlalchemy.sql.and_(*whereTerms))
946 return self._connection.execute(sql, *rows).rowcount
948 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int:
949 """Update one or more rows in a table.
951 Parameters
952 ----------
953 table : `sqlalchemy.schema.Table`
954 Table containing the rows to be updated.
955 where : `dict` [`str`, `str`]
956 A mapping from the names of columns that will be used to search for
957 existing rows to the keys that will hold these values in the
958 ``rows`` dictionaries. Note that these may not be the same due to
959 SQLAlchemy limitations.
960 *rows
961 Positional arguments are the rows to be updated. The keys in all
962 dictionaries must be the same, and may correspond to either a
963 value in the ``where`` dictionary or the name of a column to be
964 updated.
966 Raises
967 ------
968 ReadOnlyDatabaseError
969 Raised if `isWriteable` returns `False` when this method is called.
971 Returns
972 -------
973 count : `int`
974 Number of rows matched (regardless of whether the update actually
975 modified them).
977 Notes
978 -----
979 May be used inside transaction contexts, so implementations may not
980 perform operations that interrupt transactions.
982 The default implementation should be sufficient for most derived
983 classes.
984 """
985 if not self.isWriteable():
986 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.")
987 sql = table.update().where(
988 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()])
989 )
990 return self._connection.execute(sql, *rows).rowcount
992 def query(self, sql: sqlalchemy.sql.FromClause, *args, **kwds) -> sqlalchemy.engine.ResultProxy:
993 """Run a SELECT query against the database.
995 Parameters
996 ----------
997 sql : `sqlalchemy.sql.FromClause`
998 A SQLAlchemy representation of a ``SELECT`` query.
999 *args
1000 Additional positional arguments are forwarded to
1001 `sqlalchemy.engine.Connection.execute`.
1002 **kwds
1003 Additional keyword arguments are forwarded to
1004 `sqlalchemy.engine.Connection.execute`.
1006 Returns
1007 -------
1008 result : `sqlalchemy.engine.ResultProxy`
1009 Query results.
1011 Notes
1012 -----
1013 The default implementation should be sufficient for most derived
1014 classes.
1015 """
1016 # TODO: should we guard against non-SELECT queries here?
1017 return self._connection.execute(sql, *args, **kwds)
1019 origin: int
1020 """An integer ID that should be used as the default for any datasets,
1021 quanta, or other entities that use a (autoincrement, origin) compound
1022 primary key (`int`).
1023 """
1025 namespace: Optional[str]
1026 """The schema or namespace this database instance is associated with
1027 (`str` or `None`).
1028 """