Coverage for python/lsst/daf/butler/registry/interfaces/_database.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = [
24 "Database",
25 "ReadOnlyDatabaseError",
26 "DatabaseConflictError",
27 "SchemaAlreadyDefinedError",
28 "StaticTablesContext",
29]
31from abc import ABC, abstractmethod
32from contextlib import contextmanager
33from typing import (
34 Any,
35 Callable,
36 Dict,
37 Iterable,
38 Iterator,
39 List,
40 Optional,
41 Sequence,
42 Set,
43 Tuple,
44 Type,
45 Union,
46)
47import uuid
48import warnings
50import astropy.time
51import sqlalchemy
53from ...core import SpatialRegionDatabaseRepresentation, TimespanDatabaseRepresentation, ddl, time_utils
54from .._exceptions import ConflictingDefinitionError
56_IN_SAVEPOINT_TRANSACTION = "IN_SAVEPOINT_TRANSACTION"
59def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None:
60 """Test that the definition of a table in a `ddl.TableSpec` and from
61 database introspection are consistent.
63 Parameters
64 ----------
65 name : `str`
66 Name of the table (only used in error messages).
67 spec : `ddl.TableSpec`
68 Specification of the table.
69 inspection : `dict`
70 Dictionary returned by
71 `sqlalchemy.engine.reflection.Inspector.get_columns`.
73 Raises
74 ------
75 DatabaseConflictError
76 Raised if the definitions are inconsistent.
77 """
78 columnNames = [c["name"] for c in inspection]
79 if spec.fields.names != set(columnNames):
80 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; "
81 f"specification has columns {list(spec.fields.names)}, while the "
82 f"table in the database has {columnNames}.")
85class ReadOnlyDatabaseError(RuntimeError):
86 """Exception raised when a write operation is called on a read-only
87 `Database`.
88 """
91class DatabaseConflictError(ConflictingDefinitionError):
92 """Exception raised when database content (row values or schema entities)
93 are inconsistent with what this client expects.
94 """
97class SchemaAlreadyDefinedError(RuntimeError):
98 """Exception raised when trying to initialize database schema when some
99 tables already exist.
100 """
103class StaticTablesContext:
104 """Helper class used to declare the static schema for a registry layer
105 in a database.
107 An instance of this class is returned by `Database.declareStaticTables`,
108 which should be the only way it should be constructed.
109 """
111 def __init__(self, db: Database):
112 self._db = db
113 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = []
114 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection)
115 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace))
116 self._initializers: List[Callable[[Database], None]] = []
118 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
119 """Add a new table to the schema, returning its sqlalchemy
120 representation.
122 The new table may not actually be created until the end of the
123 context created by `Database.declareStaticTables`, allowing tables
124 to be declared in any order even in the presence of foreign key
125 relationships.
126 """
127 name = self._db._mangleTableName(name)
128 if name in self._tableNames:
129 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name,
130 schema=self._db.namespace))
131 table = self._db._convertTableSpec(name, spec, self._db._metadata)
132 for foreignKeySpec in spec.foreignKeys:
133 self._foreignKeys.append(
134 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata))
135 )
136 return table
138 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]:
139 """Add a named tuple of tables to the schema, returning their
140 SQLAlchemy representations in a named tuple of the same type.
142 The new tables may not actually be created until the end of the
143 context created by `Database.declareStaticTables`, allowing tables
144 to be declared in any order even in the presence of foreign key
145 relationships.
147 Notes
148 -----
149 ``specs`` *must* be an instance of a type created by
150 `collections.namedtuple`, not just regular tuple, and the returned
151 object is guaranteed to be the same. Because `~collections.namedtuple`
152 is just a factory for `type` objects, not an actual type itself,
153 we cannot represent this with type annotations.
154 """
155 return specs._make(self.addTable(name, spec) # type: ignore
156 for name, spec in zip(specs._fields, specs)) # type: ignore
158 def addInitializer(self, initializer: Callable[[Database], None]) -> None:
159 """Add a method that does one-time initialization of a database.
161 Initialization can mean anything that changes state of a database
162 and needs to be done exactly once after database schema was created.
163 An example for that could be population of schema attributes.
165 Parameters
166 ----------
167 initializer : callable
168 Method of a single argument which is a `Database` instance.
169 """
170 self._initializers.append(initializer)
173class Session:
174 """Class representing a persistent connection to a database.
176 Parameters
177 ----------
178 db : `Database`
179 Database instance.
181 Notes
182 -----
183 Instances of Session class should not be created by client code;
184 `Database.session` should be used to create context for a session::
186 with db.session() as session:
187 session.method()
188 db.method()
190 In the current implementation sessions can be nested and transactions can
191 be nested within a session. All nested sessions and transaction share the
192 same database connection.
194 Session class represents a limited subset of database API that requires
195 persistent connection to a database (e.g. temporary tables which have
196 lifetime of a session). Potentially most of the database API could be
197 associated with a Session class.
198 """
199 def __init__(self, db: Database):
200 self._db = db
202 def makeTemporaryTable(self, spec: ddl.TableSpec, name: Optional[str] = None) -> sqlalchemy.schema.Table:
203 """Create a temporary table.
205 Parameters
206 ----------
207 spec : `TableSpec`
208 Specification for the table.
209 name : `str`, optional
210 A unique (within this session/connetion) name for the table.
211 Subclasses may override to modify the actual name used. If not
212 provided, a unique name will be generated.
214 Returns
215 -------
216 table : `sqlalchemy.schema.Table`
217 SQLAlchemy representation of the table.
219 Notes
220 -----
221 Temporary tables may be created, dropped, and written to even in
222 read-only databases - at least according to the Python-level
223 protections in the `Database` classes. Server permissions may say
224 otherwise, but in that case they probably need to be modified to
225 support the full range of expected read-only butler behavior.
227 Temporary table rows are guaranteed to be dropped when a connection is
228 closed. `Database` implementations are permitted to allow the table to
229 remain as long as this is transparent to the user (i.e. "creating" the
230 temporary table in a new session should not be an error, even if it
231 does nothing).
233 It may not be possible to use temporary tables within transactions with
234 some database engines (or configurations thereof).
235 """
236 if name is None:
237 name = f"tmp_{uuid.uuid4().hex}"
238 table = self._db._convertTableSpec(name, spec, self._db._metadata, prefixes=['TEMPORARY'],
239 schema=sqlalchemy.schema.BLANK_SCHEMA)
240 if table.key in self._db._tempTables:
241 if table.key != name:
242 raise ValueError(f"A temporary table with name {name} (transformed to {table.key} by "
243 f"Database) already exists.")
244 for foreignKeySpec in spec.foreignKeys:
245 table.append_constraint(self._db._convertForeignKeySpec(name, foreignKeySpec,
246 self._db._metadata))
247 table.create(self._db._session_connection)
248 self._db._tempTables.add(table.key)
249 return table
251 def dropTemporaryTable(self, table: sqlalchemy.schema.Table) -> None:
252 """Drop a temporary table.
254 Parameters
255 ----------
256 table : `sqlalchemy.schema.Table`
257 A SQLAlchemy object returned by a previous call to
258 `makeTemporaryTable`.
259 """
260 if table.key in self._db._tempTables:
261 table.drop(self._db._session_connection)
262 self._db._tempTables.remove(table.key)
263 else:
264 raise TypeError(f"Table {table.key} was not created by makeTemporaryTable.")
267class Database(ABC):
268 """An abstract interface that represents a particular database engine's
269 representation of a single schema/namespace/database.
271 Parameters
272 ----------
273 origin : `int`
274 An integer ID that should be used as the default for any datasets,
275 quanta, or other entities that use a (autoincrement, origin) compound
276 primary key.
277 engine : `sqlalchemy.engine.Engine`
278 The SQLAlchemy engine for this `Database`.
279 namespace : `str`, optional
280 Name of the schema or namespace this instance is associated with.
281 This is passed as the ``schema`` argument when constructing a
282 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to
283 avoid confusion between "schema means namespace" and "schema means
284 table definitions".
286 Notes
287 -----
288 `Database` requires all write operations to go through its special named
289 methods. Our write patterns are sufficiently simple that we don't really
290 need the full flexibility of SQL insert/update/delete syntax, and we need
291 non-standard (but common) functionality in these operations sufficiently
292 often that it seems worthwhile to provide our own generic API.
294 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via
295 their SQLAlchemy representation) to be run, as we expect these to require
296 significantly more sophistication while still being limited to standard
297 SQL.
299 `Database` itself has several underscore-prefixed attributes:
301 - ``_engine``: SQLAlchemy object representing its engine.
302 - ``_connection``: the `sqlalchemy.engine.Connectable` object which can
303 be either an Engine or Connection if a session is active.
304 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing
305 the tables and other schema entities.
307 These are considered protected (derived classes may access them, but other
308 code should not), and read-only, aside from executing SQL via
309 ``_connection``.
310 """
312 def __init__(self, *, origin: int, engine: sqlalchemy.engine.Engine,
313 namespace: Optional[str] = None):
314 self.origin = origin
315 self.namespace = namespace
316 self._engine = engine
317 self._session_connection: Optional[sqlalchemy.engine.Connection] = None
318 self._metadata: Optional[sqlalchemy.schema.MetaData] = None
319 self._tempTables: Set[str] = set()
321 def __repr__(self) -> str:
322 # Rather than try to reproduce all the parameters used to create
323 # the object, instead report the more useful information of the
324 # connection URL.
325 uri = str(self._engine.url)
326 if self.namespace:
327 uri += f"#{self.namespace}"
328 return f'{type(self).__name__}("{uri}")'
330 @classmethod
331 def makeDefaultUri(cls, root: str) -> Optional[str]:
332 """Create a default connection URI appropriate for the given root
333 directory, or `None` if there can be no such default.
334 """
335 return None
337 @classmethod
338 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None,
339 writeable: bool = True) -> Database:
340 """Construct a database from a SQLAlchemy URI.
342 Parameters
343 ----------
344 uri : `str`
345 A SQLAlchemy URI connection string.
346 origin : `int`
347 An integer ID that should be used as the default for any datasets,
348 quanta, or other entities that use a (autoincrement, origin)
349 compound primary key.
350 namespace : `str`, optional
351 A database namespace (i.e. schema) the new instance should be
352 associated with. If `None` (default), the namespace (if any) is
353 inferred from the URI.
354 writeable : `bool`, optional
355 If `True`, allow write operations on the database, including
356 ``CREATE TABLE``.
358 Returns
359 -------
360 db : `Database`
361 A new `Database` instance.
362 """
363 return cls.fromEngine(cls.makeEngine(uri, writeable=writeable),
364 origin=origin,
365 namespace=namespace,
366 writeable=writeable)
368 @classmethod
369 @abstractmethod
370 def makeEngine(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Engine:
371 """Create a `sqlalchemy.engine.Engine` from a SQLAlchemy URI.
373 Parameters
374 ----------
375 uri : `str`
376 A SQLAlchemy URI connection string.
377 writeable : `bool`, optional
378 If `True`, allow write operations on the database, including
379 ``CREATE TABLE``.
381 Returns
382 -------
383 engine : `sqlalchemy.engine.Engine`
384 A database engine.
386 Notes
387 -----
388 Subclasses that support other ways to connect to a database are
389 encouraged to add optional arguments to their implementation of this
390 method, as long as they maintain compatibility with the base class
391 call signature.
392 """
393 raise NotImplementedError()
395 @classmethod
396 @abstractmethod
397 def fromEngine(cls, engine: sqlalchemy.engine.Engine, *, origin: int,
398 namespace: Optional[str] = None, writeable: bool = True) -> Database:
399 """Create a new `Database` from an existing `sqlalchemy.engine.Engine`.
401 Parameters
402 ----------
403 engine : `sqllachemy.engine.Engine`
404 The engine for the database. May be shared between `Database`
405 instances.
406 origin : `int`
407 An integer ID that should be used as the default for any datasets,
408 quanta, or other entities that use a (autoincrement, origin)
409 compound primary key.
410 namespace : `str`, optional
411 A different database namespace (i.e. schema) the new instance
412 should be associated with. If `None` (default), the namespace
413 (if any) is inferred from the connection.
414 writeable : `bool`, optional
415 If `True`, allow write operations on the database, including
416 ``CREATE TABLE``.
418 Returns
419 -------
420 db : `Database`
421 A new `Database` instance.
423 Notes
424 -----
425 This method allows different `Database` instances to share the same
426 engine, which is desirable when they represent different namespaces
427 can be queried together.
428 """
429 raise NotImplementedError()
431 @contextmanager
432 def session(self) -> Iterator:
433 """Return a context manager that represents a session (persistent
434 connection to a database).
435 """
436 if self._session_connection is not None:
437 # session already started, just reuse that
438 yield Session(self)
439 else:
440 # open new connection and close it when done
441 self._session_connection = self._engine.connect()
442 yield Session(self)
443 self._session_connection.close()
444 self._session_connection = None
445 # Temporary tables only live within session
446 self._tempTables = set()
448 @contextmanager
449 def transaction(self, *, interrupting: bool = False, savepoint: bool = False,
450 lock: Iterable[sqlalchemy.schema.Table] = ()) -> Iterator:
451 """Return a context manager that represents a transaction.
453 Parameters
454 ----------
455 interrupting : `bool`, optional
456 If `True` (`False` is default), this transaction block may not be
457 nested without an outer one, and attempting to do so is a logic
458 (i.e. assertion) error.
459 savepoint : `bool`, optional
460 If `True` (`False` is default), create a `SAVEPOINT`, allowing
461 exceptions raised by the database (e.g. due to constraint
462 violations) during this transaction's context to be caught outside
463 it without also rolling back all operations in an outer transaction
464 block. If `False`, transactions may still be nested, but a
465 rollback may be generated at any level and affects all levels, and
466 commits are deferred until the outermost block completes. If any
467 outer transaction block was created with ``savepoint=True``, all
468 inner blocks will be as well (regardless of the actual value
469 passed). This has no effect if this is the outermost transaction.
470 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional
471 A list of tables to lock for the duration of this transaction.
472 These locks are guaranteed to prevent concurrent writes and allow
473 this transaction (only) to acquire the same locks (others should
474 block), but only prevent concurrent reads if the database engine
475 requires that in order to block concurrent writes.
477 Notes
478 -----
479 All transactions on a connection managed by one or more `Database`
480 instances _must_ go through this method, or transaction state will not
481 be correctly managed.
482 """
483 # need a connection, use session to manage it
484 with self.session():
485 assert self._session_connection is not None
486 connection = self._session_connection
487 assert not (interrupting and connection.in_transaction()), (
488 "Logic error in transaction nesting: an operation that would "
489 "interrupt the active transaction context has been requested."
490 )
491 # We remember whether we are already in a SAVEPOINT transaction via
492 # the connection object's 'info' dict, which is explicitly for user
493 # information like this. This is safer than a regular `Database`
494 # instance attribute, because it guards against multiple `Database`
495 # instances sharing the same connection. The need to use our own
496 # flag here to track whether we're in a nested transaction should
497 # go away in SQLAlchemy 1.4, which seems to have a
498 # `Connection.in_nested_transaction()` method.
499 savepoint = savepoint or connection.info.get(_IN_SAVEPOINT_TRANSACTION, False)
500 connection.info[_IN_SAVEPOINT_TRANSACTION] = savepoint
501 if connection.in_transaction() and savepoint:
502 trans = connection.begin_nested()
503 else:
504 # Use a regular (non-savepoint) transaction always for the
505 # outermost context, as well as when a savepoint was not
506 # requested.
507 trans = connection.begin()
508 self._lockTables(lock)
509 try:
510 yield
511 trans.commit()
512 except BaseException:
513 trans.rollback()
514 raise
515 finally:
516 if not connection.in_transaction():
517 connection.info.pop(_IN_SAVEPOINT_TRANSACTION, None)
519 @property
520 def _connection(self) -> sqlalchemy.engine.Connectable:
521 """Object that can be used to execute queries
522 (`sqlalchemy.engine.Connectable`)
523 """
524 return self._session_connection or self._engine
526 @abstractmethod
527 def _lockTables(self, tables: Iterable[sqlalchemy.schema.Table] = ()) -> None:
528 """Acquire locks on the given tables.
530 This is an implementation hook for subclasses, called by `transaction`.
531 It should not be called directly by other code.
533 Parameters
534 ----------
535 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional
536 A list of tables to lock for the duration of this transaction.
537 These locks are guaranteed to prevent concurrent writes and allow
538 this transaction (only) to acquire the same locks (others should
539 block), but only prevent concurrent reads if the database engine
540 requires that in order to block concurrent writes.
541 """
542 raise NotImplementedError()
544 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool:
545 """Check whether a table is writeable, either because the database
546 connection is read-write or the table is a temporary table.
548 Parameters
549 ----------
550 table : `sqlalchemy.schema.Table`
551 SQLAlchemy table object to check.
553 Returns
554 -------
555 writeable : `bool`
556 Whether this table is writeable.
557 """
558 return self.isWriteable() or table.key in self._tempTables
560 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None:
561 """Raise if the given table is not writeable, either because the
562 database connection is read-write or the table is a temporary table.
564 Parameters
565 ----------
566 table : `sqlalchemy.schema.Table`
567 SQLAlchemy table object to check.
568 msg : `str`, optional
569 If provided, raise `ReadOnlyDatabaseError` instead of returning
570 `False`, with this message.
571 """
572 if not self.isTableWriteable(table):
573 raise ReadOnlyDatabaseError(msg)
575 @contextmanager
576 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]:
577 """Return a context manager in which the database's static DDL schema
578 can be declared.
580 Parameters
581 ----------
582 create : `bool`
583 If `True`, attempt to create all tables at the end of the context.
584 If `False`, they will be assumed to already exist.
586 Returns
587 -------
588 schema : `StaticTablesContext`
589 A helper object that is used to add new tables.
591 Raises
592 ------
593 ReadOnlyDatabaseError
594 Raised if ``create`` is `True`, `Database.isWriteable` is `False`,
595 and one or more declared tables do not already exist.
597 Examples
598 --------
599 Given a `Database` instance ``db``::
601 with db.declareStaticTables(create=True) as schema:
602 schema.addTable("table1", TableSpec(...))
603 schema.addTable("table2", TableSpec(...))
605 Notes
606 -----
607 A database's static DDL schema must be declared before any dynamic
608 tables are managed via calls to `ensureTableExists` or
609 `getExistingTable`. The order in which static schema tables are added
610 inside the context block is unimportant; they will automatically be
611 sorted and added in an order consistent with their foreign key
612 relationships.
613 """
614 if create and not self.isWriteable():
615 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.")
616 self._metadata = sqlalchemy.MetaData(schema=self.namespace)
617 try:
618 context = StaticTablesContext(self)
619 if create and context._tableNames:
620 # Looks like database is already initalized, to avoid danger
621 # of modifying/destroying valid schema we refuse to do
622 # anything in this case
623 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.")
624 yield context
625 for table, foreignKey in context._foreignKeys:
626 table.append_constraint(foreignKey)
627 if create:
628 if self.namespace is not None:
629 if self.namespace not in context._inspector.get_schema_names():
630 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace))
631 # In our tables we have columns that make use of sqlalchemy
632 # Sequence objects. There is currently a bug in sqlalchemy that
633 # causes a deprecation warning to be thrown on a property of
634 # the Sequence object when the repr for the sequence is
635 # created. Here a filter is used to catch these deprecation
636 # warnings when tables are created.
637 with warnings.catch_warnings():
638 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning)
639 self._metadata.create_all(self._connection)
640 # call all initializer methods sequentially
641 for init in context._initializers:
642 init(self)
643 except BaseException:
644 self._metadata = None
645 raise
647 @abstractmethod
648 def isWriteable(self) -> bool:
649 """Return `True` if this database can be modified by this client.
650 """
651 raise NotImplementedError()
653 @abstractmethod
654 def __str__(self) -> str:
655 """Return a human-readable identifier for this `Database`, including
656 any namespace or schema that identifies its names within a `Registry`.
657 """
658 raise NotImplementedError()
660 @property
661 def dialect(self) -> sqlalchemy.engine.Dialect:
662 """The SQLAlchemy dialect for this database engine
663 (`sqlalchemy.engine.Dialect`).
664 """
665 return self._engine.dialect
667 def shrinkDatabaseEntityName(self, original: str) -> str:
668 """Return a version of the given name that fits within this database
669 engine's length limits for table, constraint, indexes, and sequence
670 names.
672 Implementations should not assume that simple truncation is safe,
673 because multiple long names often begin with the same prefix.
675 The default implementation simply returns the given name.
677 Parameters
678 ----------
679 original : `str`
680 The original name.
682 Returns
683 -------
684 shrunk : `str`
685 The new, possibly shortened name.
686 """
687 return original
689 def expandDatabaseEntityName(self, shrunk: str) -> str:
690 """Retrieve the original name for a database entity that was too long
691 to fit within the database engine's limits.
693 Parameters
694 ----------
695 original : `str`
696 The original name.
698 Returns
699 -------
700 shrunk : `str`
701 The new, possibly shortened name.
702 """
703 return shrunk
705 def _mangleTableName(self, name: str) -> str:
706 """Map a logical, user-visible table name to the true table name used
707 in the database.
709 The default implementation returns the given name unchanged.
711 Parameters
712 ----------
713 name : `str`
714 Input table name. Should not include a namespace (i.e. schema)
715 prefix.
717 Returns
718 -------
719 mangled : `str`
720 Mangled version of the table name (still with no namespace prefix).
722 Notes
723 -----
724 Reimplementations of this method must be idempotent - mangling an
725 already-mangled name must have no effect.
726 """
727 return name
729 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]:
730 """Create constraints based on this spec.
732 Parameters
733 ----------
734 table : `str`
735 Name of the table this column is being added to.
736 spec : `FieldSpec`
737 Specification for the field to be added.
739 Returns
740 -------
741 constraint : `list` of `sqlalchemy.CheckConstraint`
742 Constraint added for this column.
743 """
744 # By default we return no additional constraints
745 return []
747 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData,
748 **kwds: Any) -> sqlalchemy.schema.Column:
749 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`.
751 Parameters
752 ----------
753 table : `str`
754 Name of the table this column is being added to.
755 spec : `FieldSpec`
756 Specification for the field to be added.
757 metadata : `sqlalchemy.MetaData`
758 SQLAlchemy representation of the DDL schema this field's table is
759 being added to.
760 **kwds
761 Additional keyword arguments to forward to the
762 `sqlalchemy.schema.Column` constructor. This is provided to make
763 it easier for derived classes to delegate to ``super()`` while
764 making only minor changes.
766 Returns
767 -------
768 column : `sqlalchemy.schema.Column`
769 SQLAlchemy representation of the field.
770 """
771 args = [spec.name, spec.getSizedColumnType()]
772 if spec.autoincrement:
773 # Generate a sequence to use for auto incrementing for databases
774 # that do not support it natively. This will be ignored by
775 # sqlalchemy for databases that do support it.
776 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"),
777 metadata=metadata))
778 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}."
779 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey,
780 comment=spec.doc, server_default=spec.default, **kwds)
782 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData,
783 **kwds: Any) -> sqlalchemy.schema.ForeignKeyConstraint:
784 """Convert a `ForeignKeySpec` to a
785 `sqlalchemy.schema.ForeignKeyConstraint`.
787 Parameters
788 ----------
789 table : `str`
790 Name of the table this foreign key is being added to.
791 spec : `ForeignKeySpec`
792 Specification for the foreign key to be added.
793 metadata : `sqlalchemy.MetaData`
794 SQLAlchemy representation of the DDL schema this constraint is
795 being added to.
796 **kwds
797 Additional keyword arguments to forward to the
798 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is
799 provided to make it easier for derived classes to delegate to
800 ``super()`` while making only minor changes.
802 Returns
803 -------
804 constraint : `sqlalchemy.schema.ForeignKeyConstraint`
805 SQLAlchemy representation of the constraint.
806 """
807 name = self.shrinkDatabaseEntityName(
808 "_".join(["fkey", table, self._mangleTableName(spec.table)]
809 + list(spec.target) + list(spec.source))
810 )
811 return sqlalchemy.schema.ForeignKeyConstraint(
812 spec.source,
813 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target],
814 name=name,
815 ondelete=spec.onDelete
816 )
818 def _convertExclusionConstraintSpec(self, table: str,
819 spec: Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...],
820 metadata: sqlalchemy.MetaData) -> sqlalchemy.schema.Constraint:
821 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy
822 constraint representation.
824 Parameters
825 ----------
826 table : `str`
827 Name of the table this constraint is being added to.
828 spec : `tuple` [ `str` or `type` ]
829 A tuple of `str` column names and the `type` object returned by
830 `getTimespanRepresentation` (which must appear exactly once),
831 indicating the order of the columns in the index used to back the
832 constraint.
833 metadata : `sqlalchemy.MetaData`
834 SQLAlchemy representation of the DDL schema this constraint is
835 being added to.
837 Returns
838 -------
839 constraint : `sqlalchemy.schema.Constraint`
840 SQLAlchemy representation of the constraint.
842 Raises
843 ------
844 NotImplementedError
845 Raised if this database does not support exclusion constraints.
846 """
847 raise NotImplementedError(f"Database {self} does not support exclusion constraints.")
849 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData,
850 **kwds: Any) -> sqlalchemy.schema.Table:
851 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`.
853 Parameters
854 ----------
855 spec : `TableSpec`
856 Specification for the foreign key to be added.
857 metadata : `sqlalchemy.MetaData`
858 SQLAlchemy representation of the DDL schema this table is being
859 added to.
860 **kwds
861 Additional keyword arguments to forward to the
862 `sqlalchemy.schema.Table` constructor. This is provided to make it
863 easier for derived classes to delegate to ``super()`` while making
864 only minor changes.
866 Returns
867 -------
868 table : `sqlalchemy.schema.Table`
869 SQLAlchemy representation of the table.
871 Notes
872 -----
873 This method does not handle ``spec.foreignKeys`` at all, in order to
874 avoid circular dependencies. These are added by higher-level logic in
875 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`.
876 """
877 name = self._mangleTableName(name)
878 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields]
880 # Add any column constraints
881 for fieldSpec in spec.fields:
882 args.extend(self._makeColumnConstraints(name, fieldSpec))
884 # Track indexes added for primary key and unique constraints, to make
885 # sure we don't add duplicate explicit or foreign key indexes for
886 # those.
887 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)}
888 args.extend(
889 sqlalchemy.schema.UniqueConstraint(
890 *columns,
891 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns)))
892 )
893 for columns in spec.unique
894 )
895 allIndexes.update(spec.unique)
896 args.extend(
897 sqlalchemy.schema.Index(
898 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))),
899 *columns,
900 unique=(columns in spec.unique)
901 )
902 for columns in spec.indexes if columns not in allIndexes
903 )
904 allIndexes.update(spec.indexes)
905 args.extend(
906 sqlalchemy.schema.Index(
907 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)),
908 *fk.source,
909 )
910 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes
911 )
913 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion)
915 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}."
916 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds)
918 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
919 """Ensure that a table with the given name and specification exists,
920 creating it if necessary.
922 Parameters
923 ----------
924 name : `str`
925 Name of the table (not including namespace qualifiers).
926 spec : `TableSpec`
927 Specification for the table. This will be used when creating the
928 table, and *may* be used when obtaining an existing table to check
929 for consistency, but no such check is guaranteed.
931 Returns
932 -------
933 table : `sqlalchemy.schema.Table`
934 SQLAlchemy representation of the table.
936 Raises
937 ------
938 ReadOnlyDatabaseError
939 Raised if `isWriteable` returns `False`, and the table does not
940 already exist.
941 DatabaseConflictError
942 Raised if the table exists but ``spec`` is inconsistent with its
943 definition.
945 Notes
946 -----
947 This method may not be called within transactions. It may be called on
948 read-only databases if and only if the table does in fact already
949 exist.
951 Subclasses may override this method, but usually should not need to.
952 """
953 # TODO: if _engine is used to make a table then it uses separate
954 # connection and should not interfere with current transaction
955 assert self._session_connection is None or not self._session_connection.in_transaction(), \
956 "Table creation interrupts transactions."
957 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
958 table = self.getExistingTable(name, spec)
959 if table is not None:
960 return table
961 if not self.isWriteable():
962 raise ReadOnlyDatabaseError(
963 f"Table {name} does not exist, and cannot be created "
964 f"because database {self} is read-only."
965 )
966 table = self._convertTableSpec(name, spec, self._metadata)
967 for foreignKeySpec in spec.foreignKeys:
968 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
969 table.create(self._connection)
970 return table
972 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]:
973 """Obtain an existing table with the given name and specification.
975 Parameters
976 ----------
977 name : `str`
978 Name of the table (not including namespace qualifiers).
979 spec : `TableSpec`
980 Specification for the table. This will be used when creating the
981 SQLAlchemy representation of the table, and it is used to
982 check that the actual table in the database is consistent.
984 Returns
985 -------
986 table : `sqlalchemy.schema.Table` or `None`
987 SQLAlchemy representation of the table, or `None` if it does not
988 exist.
990 Raises
991 ------
992 DatabaseConflictError
993 Raised if the table exists but ``spec`` is inconsistent with its
994 definition.
996 Notes
997 -----
998 This method can be called within transactions and never modifies the
999 database.
1001 Subclasses may override this method, but usually should not need to.
1002 """
1003 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
1004 name = self._mangleTableName(name)
1005 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}")
1006 if table is not None:
1007 if spec.fields.names != set(table.columns.keys()):
1008 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new "
1009 f"specification has columns {list(spec.fields.names)}, while "
1010 f"the previous definition has {list(table.columns.keys())}.")
1011 else:
1012 inspector = sqlalchemy.engine.reflection.Inspector(self._connection)
1013 if name in inspector.get_table_names(schema=self.namespace):
1014 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace))
1015 table = self._convertTableSpec(name, spec, self._metadata)
1016 for foreignKeySpec in spec.foreignKeys:
1017 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
1018 return table
1019 return table
1021 @classmethod
1022 def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]:
1023 """Return a `type` that encapsulates the way `Timespan` objects are
1024 stored in this database.
1026 `Database` does not automatically use the return type of this method
1027 anywhere else; calling code is responsible for making sure that DDL
1028 and queries are consistent with it.
1030 Returns
1031 -------
1032 TimespanReprClass : `type` (`TimespanDatabaseRepresention` subclass)
1033 A type that encapsulates the way `Timespan` objects should be
1034 stored in this database.
1036 Notes
1037 -----
1038 There are two big reasons we've decided to keep timespan-mangling logic
1039 outside the `Database` implementations, even though the choice of
1040 representation is ultimately up to a `Database` implementation:
1042 - Timespans appear in relatively few tables and queries in our
1043 typical usage, and the code that operates on them is already aware
1044 that it is working with timespans. In contrast, a
1045 timespan-representation-aware implementation of, say, `insert`,
1046 would need to have extra logic to identify when timespan-mangling
1047 needed to occur, which would usually be useless overhead.
1049 - SQLAlchemy's rich SELECT query expression system has no way to wrap
1050 multiple columns in a single expression object (the ORM does, but
1051 we are not using the ORM). So we would have to wrap _much_ more of
1052 that code in our own interfaces to encapsulate timespan
1053 representations there.
1054 """
1055 return TimespanDatabaseRepresentation.Compound
1057 @classmethod
1058 def getSpatialRegionRepresentation(cls) -> Type[SpatialRegionDatabaseRepresentation]:
1059 """Return a `type` that encapsulates the way `lsst.sphgeom.Region`
1060 objects are stored in this database.
1062 `Database` does not automatically use the return type of this method
1063 anywhere else; calling code is responsible for making sure that DDL
1064 and queries are consistent with it.
1066 Returns
1067 -------
1068 RegionReprClass : `type` (`SpatialRegionDatabaseRepresention` subclass)
1069 A type that encapsulates the way `lsst.sphgeom.Region` objects
1070 should be stored in this database.
1072 Notes
1073 -----
1074 See `getTimespanRepresentation` for comments on why this method is not
1075 more tightly integrated with the rest of the `Database` interface.
1076 """
1077 return SpatialRegionDatabaseRepresentation
1079 def sync(self, table: sqlalchemy.schema.Table, *,
1080 keys: Dict[str, Any],
1081 compared: Optional[Dict[str, Any]] = None,
1082 extra: Optional[Dict[str, Any]] = None,
1083 returning: Optional[Sequence[str]] = None,
1084 ) -> Tuple[Optional[Dict[str, Any]], bool]:
1085 """Insert into a table as necessary to ensure database contains
1086 values equivalent to the given ones.
1088 Parameters
1089 ----------
1090 table : `sqlalchemy.schema.Table`
1091 Table to be queried and possibly inserted into.
1092 keys : `dict`
1093 Column name-value pairs used to search for an existing row; must
1094 be a combination that can be used to select a single row if one
1095 exists. If such a row does not exist, these values are used in
1096 the insert.
1097 compared : `dict`, optional
1098 Column name-value pairs that are compared to those in any existing
1099 row. If such a row does not exist, these rows are used in the
1100 insert.
1101 extra : `dict`, optional
1102 Column name-value pairs that are ignored if a matching row exists,
1103 but used in an insert if one is necessary.
1104 returning : `~collections.abc.Sequence` of `str`, optional
1105 The names of columns whose values should be returned.
1107 Returns
1108 -------
1109 row : `dict`, optional
1110 The value of the fields indicated by ``returning``, or `None` if
1111 ``returning`` is `None`.
1112 inserted : `bool`
1113 If `True`, a new row was inserted.
1115 Raises
1116 ------
1117 DatabaseConflictError
1118 Raised if the values in ``compared`` do not match the values in the
1119 database.
1120 ReadOnlyDatabaseError
1121 Raised if `isWriteable` returns `False`, and no matching record
1122 already exists.
1124 Notes
1125 -----
1126 May be used inside transaction contexts, so implementations may not
1127 perform operations that interrupt transactions.
1129 It may be called on read-only databases if and only if the matching row
1130 does in fact already exist.
1131 """
1133 def check() -> Tuple[int, Optional[List[str]], Optional[List]]:
1134 """Query for a row that matches the ``key`` argument, and compare
1135 to what was given by the caller.
1137 Returns
1138 -------
1139 n : `int`
1140 Number of matching rows. ``n != 1`` is always an error, but
1141 it's a different kind of error depending on where `check` is
1142 being called.
1143 bad : `list` of `str`, or `None`
1144 The subset of the keys of ``compared`` for which the existing
1145 values did not match the given one. Once again, ``not bad``
1146 is always an error, but a different kind on context. `None`
1147 if ``n != 1``
1148 result : `list` or `None`
1149 Results in the database that correspond to the columns given
1150 in ``returning``, or `None` if ``returning is None``.
1151 """
1152 toSelect: Set[str] = set()
1153 if compared is not None:
1154 toSelect.update(compared.keys())
1155 if returning is not None:
1156 toSelect.update(returning)
1157 if not toSelect:
1158 # Need to select some column, even if we just want to see
1159 # how many rows we get back.
1160 toSelect.add(next(iter(keys.keys())))
1161 selectSql = sqlalchemy.sql.select(
1162 [table.columns[k].label(k) for k in toSelect]
1163 ).select_from(table).where(
1164 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])
1165 )
1166 fetched = list(self._connection.execute(selectSql).fetchall())
1167 if len(fetched) != 1:
1168 return len(fetched), None, None
1169 existing = fetched[0]
1170 if compared is not None:
1172 def safeNotEqual(a: Any, b: Any) -> bool:
1173 if isinstance(a, astropy.time.Time):
1174 return not time_utils.TimeConverter().times_equal(a, b)
1175 return a != b
1177 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}"
1178 for k, v in compared.items()
1179 if safeNotEqual(existing[k], v)]
1180 else:
1181 inconsistencies = []
1182 if returning is not None:
1183 toReturn: Optional[list] = [existing[k] for k in returning]
1184 else:
1185 toReturn = None
1186 return 1, inconsistencies, toReturn
1188 if self.isTableWriteable(table):
1189 # Try an insert first, but allow it to fail (in only specific
1190 # ways).
1191 row = keys.copy()
1192 if compared is not None:
1193 row.update(compared)
1194 if extra is not None:
1195 row.update(extra)
1196 with self.transaction(lock=[table]):
1197 inserted = bool(self.ensure(table, row))
1198 # Need to perform check() for this branch inside the
1199 # transaction, so we roll back an insert that didn't do
1200 # what we expected. That limits the extent to which we
1201 # can reduce duplication between this block and the other
1202 # ones that perform similar logic.
1203 n, bad, result = check()
1204 if n < 1:
1205 raise RuntimeError(
1206 "Necessary insertion in sync did not seem to affect table. This is a bug."
1207 )
1208 elif n > 1:
1209 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
1210 f"unique constraint for table {table.name}.")
1211 elif bad:
1212 if inserted:
1213 raise RuntimeError(
1214 f"Conflict ({bad}) in sync after successful insert; this is "
1215 "possible if the same table is being updated by a concurrent "
1216 "process that isn't using sync, but it may also be a bug in "
1217 "daf_butler."
1218 )
1219 else:
1220 raise DatabaseConflictError(
1221 f"Conflict in sync for table {table.name} on column(s) {bad}."
1222 )
1223 else:
1224 # Database is not writeable; just see if the row exists.
1225 n, bad, result = check()
1226 if n < 1:
1227 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.")
1228 elif n > 1:
1229 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.")
1230 elif bad:
1231 raise DatabaseConflictError(
1232 f"Conflict in sync for table {table.name} on column(s) {bad}."
1233 )
1234 inserted = False
1235 if returning is None:
1236 return None, inserted
1237 else:
1238 assert result is not None
1239 return {k: v for k, v in zip(returning, result)}, inserted
1241 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False,
1242 select: Optional[sqlalchemy.sql.Select] = None,
1243 names: Optional[Iterable[str]] = None,
1244 ) -> Optional[List[int]]:
1245 """Insert one or more rows into a table, optionally returning
1246 autoincrement primary key values.
1248 Parameters
1249 ----------
1250 table : `sqlalchemy.schema.Table`
1251 Table rows should be inserted into.
1252 returnIds: `bool`
1253 If `True` (`False` is default), return the values of the table's
1254 autoincrement primary key field (which much exist).
1255 select : `sqlalchemy.sql.Select`, optional
1256 A SELECT query expression to insert rows from. Cannot be provided
1257 with either ``rows`` or ``returnIds=True``.
1258 names : `Iterable` [ `str` ], optional
1259 Names of columns in ``table`` to be populated, ordered to match the
1260 columns returned by ``select``. Ignored if ``select`` is `None`.
1261 If not provided, the columns returned by ``select`` must be named
1262 to match the desired columns of ``table``.
1263 *rows
1264 Positional arguments are the rows to be inserted, as dictionaries
1265 mapping column name to value. The keys in all dictionaries must
1266 be the same.
1268 Returns
1269 -------
1270 ids : `None`, or `list` of `int`
1271 If ``returnIds`` is `True`, a `list` containing the inserted
1272 values for the table's autoincrement primary key.
1274 Raises
1275 ------
1276 ReadOnlyDatabaseError
1277 Raised if `isWriteable` returns `False` when this method is called.
1279 Notes
1280 -----
1281 The default implementation uses bulk insert syntax when ``returnIds``
1282 is `False`, and a loop over single-row insert operations when it is
1283 `True`.
1285 Derived classes should reimplement when they can provide a more
1286 efficient implementation (especially for the latter case).
1288 May be used inside transaction contexts, so implementations may not
1289 perform operations that interrupt transactions.
1290 """
1291 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.")
1292 if select is not None and (rows or returnIds):
1293 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.")
1294 if not rows and select is None:
1295 if returnIds:
1296 return []
1297 else:
1298 return None
1299 if not returnIds:
1300 if select is not None:
1301 if names is None:
1302 names = select.columns.keys()
1303 self._connection.execute(table.insert().from_select(names, select))
1304 else:
1305 self._connection.execute(table.insert(), *rows)
1306 return None
1307 else:
1308 sql = table.insert()
1309 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows]
1311 @abstractmethod
1312 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None:
1313 """Insert one or more rows into a table, replacing any existing rows
1314 for which insertion of a new row would violate the primary key
1315 constraint.
1317 Parameters
1318 ----------
1319 table : `sqlalchemy.schema.Table`
1320 Table rows should be inserted into.
1321 *rows
1322 Positional arguments are the rows to be inserted, as dictionaries
1323 mapping column name to value. The keys in all dictionaries must
1324 be the same.
1326 Raises
1327 ------
1328 ReadOnlyDatabaseError
1329 Raised if `isWriteable` returns `False` when this method is called.
1331 Notes
1332 -----
1333 May be used inside transaction contexts, so implementations may not
1334 perform operations that interrupt transactions.
1336 Implementations should raise a `sqlalchemy.exc.IntegrityError`
1337 exception when a constraint other than the primary key would be
1338 violated.
1340 Implementations are not required to support `replace` on tables
1341 with autoincrement keys.
1342 """
1343 raise NotImplementedError()
1345 @abstractmethod
1346 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict) -> int:
1347 """Insert one or more rows into a table, skipping any rows for which
1348 insertion would violate any constraint.
1350 Parameters
1351 ----------
1352 table : `sqlalchemy.schema.Table`
1353 Table rows should be inserted into.
1354 *rows
1355 Positional arguments are the rows to be inserted, as dictionaries
1356 mapping column name to value. The keys in all dictionaries must
1357 be the same.
1359 Returns
1360 -------
1361 count : `int`
1362 The number of rows actually inserted.
1364 Raises
1365 ------
1366 ReadOnlyDatabaseError
1367 Raised if `isWriteable` returns `False` when this method is called.
1368 This is raised even if the operation would do nothing even on a
1369 writeable database.
1371 Notes
1372 -----
1373 May be used inside transaction contexts, so implementations may not
1374 perform operations that interrupt transactions.
1376 Implementations are not required to support `ensure` on tables
1377 with autoincrement keys.
1378 """
1379 raise NotImplementedError()
1381 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int:
1382 """Delete one or more rows from a table.
1384 Parameters
1385 ----------
1386 table : `sqlalchemy.schema.Table`
1387 Table that rows should be deleted from.
1388 columns: `~collections.abc.Iterable` of `str`
1389 The names of columns that will be used to constrain the rows to
1390 be deleted; these will be combined via ``AND`` to form the
1391 ``WHERE`` clause of the delete query.
1392 *rows
1393 Positional arguments are the keys of rows to be deleted, as
1394 dictionaries mapping column name to value. The keys in all
1395 dictionaries must exactly the names in ``columns``.
1397 Returns
1398 -------
1399 count : `int`
1400 Number of rows deleted.
1402 Raises
1403 ------
1404 ReadOnlyDatabaseError
1405 Raised if `isWriteable` returns `False` when this method is called.
1407 Notes
1408 -----
1409 May be used inside transaction contexts, so implementations may not
1410 perform operations that interrupt transactions.
1412 The default implementation should be sufficient for most derived
1413 classes.
1414 """
1415 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.")
1416 if columns and not rows:
1417 # If there are no columns, this operation is supposed to delete
1418 # everything (so we proceed as usual). But if there are columns,
1419 # but no rows, it was a constrained bulk operation where the
1420 # constraint is that no rows match, and we should short-circuit
1421 # while reporting that no rows were affected.
1422 return 0
1423 sql = table.delete()
1424 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns]
1425 if whereTerms:
1426 sql = sql.where(sqlalchemy.sql.and_(*whereTerms))
1427 return self._connection.execute(sql, *rows).rowcount
1429 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int:
1430 """Update one or more rows in a table.
1432 Parameters
1433 ----------
1434 table : `sqlalchemy.schema.Table`
1435 Table containing the rows to be updated.
1436 where : `dict` [`str`, `str`]
1437 A mapping from the names of columns that will be used to search for
1438 existing rows to the keys that will hold these values in the
1439 ``rows`` dictionaries. Note that these may not be the same due to
1440 SQLAlchemy limitations.
1441 *rows
1442 Positional arguments are the rows to be updated. The keys in all
1443 dictionaries must be the same, and may correspond to either a
1444 value in the ``where`` dictionary or the name of a column to be
1445 updated.
1447 Returns
1448 -------
1449 count : `int`
1450 Number of rows matched (regardless of whether the update actually
1451 modified them).
1453 Raises
1454 ------
1455 ReadOnlyDatabaseError
1456 Raised if `isWriteable` returns `False` when this method is called.
1458 Notes
1459 -----
1460 May be used inside transaction contexts, so implementations may not
1461 perform operations that interrupt transactions.
1463 The default implementation should be sufficient for most derived
1464 classes.
1465 """
1466 self.assertTableWriteable(table, f"Cannot update read-only table {table}.")
1467 if not rows:
1468 return 0
1469 sql = table.update().where(
1470 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()])
1471 )
1472 return self._connection.execute(sql, *rows).rowcount
1474 def query(self, sql: sqlalchemy.sql.FromClause,
1475 *args: Any, **kwds: Any) -> sqlalchemy.engine.ResultProxy:
1476 """Run a SELECT query against the database.
1478 Parameters
1479 ----------
1480 sql : `sqlalchemy.sql.FromClause`
1481 A SQLAlchemy representation of a ``SELECT`` query.
1482 *args
1483 Additional positional arguments are forwarded to
1484 `sqlalchemy.engine.Connection.execute`.
1485 **kwds
1486 Additional keyword arguments are forwarded to
1487 `sqlalchemy.engine.Connection.execute`.
1489 Returns
1490 -------
1491 result : `sqlalchemy.engine.ResultProxy`
1492 Query results.
1494 Notes
1495 -----
1496 The default implementation should be sufficient for most derived
1497 classes.
1498 """
1499 # TODO: should we guard against non-SELECT queries here?
1500 return self._connection.execute(sql, *args, **kwds)
1502 origin: int
1503 """An integer ID that should be used as the default for any datasets,
1504 quanta, or other entities that use a (autoincrement, origin) compound
1505 primary key (`int`).
1506 """
1508 namespace: Optional[str]
1509 """The schema or namespace this database instance is associated with
1510 (`str` or `None`).
1511 """