Coverage for python/lsst/daf/butler/registry/interfaces/_database.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = [
24 "Database",
25 "ReadOnlyDatabaseError",
26 "DatabaseConflictError",
27 "SchemaAlreadyDefinedError",
28 "StaticTablesContext",
29]
31from abc import ABC, abstractmethod
32from collections import defaultdict
33from contextlib import contextmanager
34from typing import (
35 Any,
36 Callable,
37 Dict,
38 Iterable,
39 Iterator,
40 List,
41 Optional,
42 Sequence,
43 Set,
44 Tuple,
45 Type,
46 Union,
47)
48import uuid
49import warnings
51import astropy.time
52import sqlalchemy
54from ...core import SpatialRegionDatabaseRepresentation, TimespanDatabaseRepresentation, ddl, time_utils
55from .._exceptions import ConflictingDefinitionError
57_IN_SAVEPOINT_TRANSACTION = "IN_SAVEPOINT_TRANSACTION"
60def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None:
61 """Test that the definition of a table in a `ddl.TableSpec` and from
62 database introspection are consistent.
64 Parameters
65 ----------
66 name : `str`
67 Name of the table (only used in error messages).
68 spec : `ddl.TableSpec`
69 Specification of the table.
70 inspection : `dict`
71 Dictionary returned by
72 `sqlalchemy.engine.reflection.Inspector.get_columns`.
74 Raises
75 ------
76 DatabaseConflictError
77 Raised if the definitions are inconsistent.
78 """
79 columnNames = [c["name"] for c in inspection]
80 if spec.fields.names != set(columnNames):
81 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; "
82 f"specification has columns {list(spec.fields.names)}, while the "
83 f"table in the database has {columnNames}.")
86class ReadOnlyDatabaseError(RuntimeError):
87 """Exception raised when a write operation is called on a read-only
88 `Database`.
89 """
92class DatabaseConflictError(ConflictingDefinitionError):
93 """Exception raised when database content (row values or schema entities)
94 are inconsistent with what this client expects.
95 """
98class SchemaAlreadyDefinedError(RuntimeError):
99 """Exception raised when trying to initialize database schema when some
100 tables already exist.
101 """
104class StaticTablesContext:
105 """Helper class used to declare the static schema for a registry layer
106 in a database.
108 An instance of this class is returned by `Database.declareStaticTables`,
109 which should be the only way it should be constructed.
110 """
112 def __init__(self, db: Database):
113 self._db = db
114 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = []
115 self._inspector = sqlalchemy.inspect(self._db._connection)
116 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace))
117 self._initializers: List[Callable[[Database], None]] = []
119 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
120 """Add a new table to the schema, returning its sqlalchemy
121 representation.
123 The new table may not actually be created until the end of the
124 context created by `Database.declareStaticTables`, allowing tables
125 to be declared in any order even in the presence of foreign key
126 relationships.
127 """
128 name = self._db._mangleTableName(name)
129 if name in self._tableNames:
130 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name,
131 schema=self._db.namespace))
132 table = self._db._convertTableSpec(name, spec, self._db._metadata)
133 for foreignKeySpec in spec.foreignKeys:
134 self._foreignKeys.append(
135 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata))
136 )
137 return table
139 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]:
140 """Add a named tuple of tables to the schema, returning their
141 SQLAlchemy representations in a named tuple of the same type.
143 The new tables may not actually be created until the end of the
144 context created by `Database.declareStaticTables`, allowing tables
145 to be declared in any order even in the presence of foreign key
146 relationships.
148 Notes
149 -----
150 ``specs`` *must* be an instance of a type created by
151 `collections.namedtuple`, not just regular tuple, and the returned
152 object is guaranteed to be the same. Because `~collections.namedtuple`
153 is just a factory for `type` objects, not an actual type itself,
154 we cannot represent this with type annotations.
155 """
156 return specs._make(self.addTable(name, spec) # type: ignore
157 for name, spec in zip(specs._fields, specs)) # type: ignore
159 def addInitializer(self, initializer: Callable[[Database], None]) -> None:
160 """Add a method that does one-time initialization of a database.
162 Initialization can mean anything that changes state of a database
163 and needs to be done exactly once after database schema was created.
164 An example for that could be population of schema attributes.
166 Parameters
167 ----------
168 initializer : callable
169 Method of a single argument which is a `Database` instance.
170 """
171 self._initializers.append(initializer)
174class Session:
175 """Class representing a persistent connection to a database.
177 Parameters
178 ----------
179 db : `Database`
180 Database instance.
182 Notes
183 -----
184 Instances of Session class should not be created by client code;
185 `Database.session` should be used to create context for a session::
187 with db.session() as session:
188 session.method()
189 db.method()
191 In the current implementation sessions can be nested and transactions can
192 be nested within a session. All nested sessions and transaction share the
193 same database connection.
195 Session class represents a limited subset of database API that requires
196 persistent connection to a database (e.g. temporary tables which have
197 lifetime of a session). Potentially most of the database API could be
198 associated with a Session class.
199 """
200 def __init__(self, db: Database):
201 self._db = db
203 def makeTemporaryTable(self, spec: ddl.TableSpec, name: Optional[str] = None) -> sqlalchemy.schema.Table:
204 """Create a temporary table.
206 Parameters
207 ----------
208 spec : `TableSpec`
209 Specification for the table.
210 name : `str`, optional
211 A unique (within this session/connetion) name for the table.
212 Subclasses may override to modify the actual name used. If not
213 provided, a unique name will be generated.
215 Returns
216 -------
217 table : `sqlalchemy.schema.Table`
218 SQLAlchemy representation of the table.
220 Notes
221 -----
222 Temporary tables may be created, dropped, and written to even in
223 read-only databases - at least according to the Python-level
224 protections in the `Database` classes. Server permissions may say
225 otherwise, but in that case they probably need to be modified to
226 support the full range of expected read-only butler behavior.
228 Temporary table rows are guaranteed to be dropped when a connection is
229 closed. `Database` implementations are permitted to allow the table to
230 remain as long as this is transparent to the user (i.e. "creating" the
231 temporary table in a new session should not be an error, even if it
232 does nothing).
234 It may not be possible to use temporary tables within transactions with
235 some database engines (or configurations thereof).
236 """
237 if name is None:
238 name = f"tmp_{uuid.uuid4().hex}"
239 table = self._db._convertTableSpec(name, spec, self._db._metadata, prefixes=['TEMPORARY'],
240 schema=sqlalchemy.schema.BLANK_SCHEMA)
241 if table.key in self._db._tempTables:
242 if table.key != name:
243 raise ValueError(f"A temporary table with name {name} (transformed to {table.key} by "
244 f"Database) already exists.")
245 for foreignKeySpec in spec.foreignKeys:
246 table.append_constraint(self._db._convertForeignKeySpec(name, foreignKeySpec,
247 self._db._metadata))
248 table.create(self._db._session_connection)
249 self._db._tempTables.add(table.key)
250 return table
252 def dropTemporaryTable(self, table: sqlalchemy.schema.Table) -> None:
253 """Drop a temporary table.
255 Parameters
256 ----------
257 table : `sqlalchemy.schema.Table`
258 A SQLAlchemy object returned by a previous call to
259 `makeTemporaryTable`.
260 """
261 if table.key in self._db._tempTables:
262 table.drop(self._db._session_connection)
263 self._db._tempTables.remove(table.key)
264 else:
265 raise TypeError(f"Table {table.key} was not created by makeTemporaryTable.")
268class Database(ABC):
269 """An abstract interface that represents a particular database engine's
270 representation of a single schema/namespace/database.
272 Parameters
273 ----------
274 origin : `int`
275 An integer ID that should be used as the default for any datasets,
276 quanta, or other entities that use a (autoincrement, origin) compound
277 primary key.
278 engine : `sqlalchemy.engine.Engine`
279 The SQLAlchemy engine for this `Database`.
280 namespace : `str`, optional
281 Name of the schema or namespace this instance is associated with.
282 This is passed as the ``schema`` argument when constructing a
283 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to
284 avoid confusion between "schema means namespace" and "schema means
285 table definitions".
287 Notes
288 -----
289 `Database` requires all write operations to go through its special named
290 methods. Our write patterns are sufficiently simple that we don't really
291 need the full flexibility of SQL insert/update/delete syntax, and we need
292 non-standard (but common) functionality in these operations sufficiently
293 often that it seems worthwhile to provide our own generic API.
295 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via
296 their SQLAlchemy representation) to be run, as we expect these to require
297 significantly more sophistication while still being limited to standard
298 SQL.
300 `Database` itself has several underscore-prefixed attributes:
302 - ``_engine``: SQLAlchemy object representing its engine.
303 - ``_connection``: the `sqlalchemy.engine.Connectable` object which can
304 be either an Engine or Connection if a session is active.
305 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing
306 the tables and other schema entities.
308 These are considered protected (derived classes may access them, but other
309 code should not), and read-only, aside from executing SQL via
310 ``_connection``.
311 """
313 def __init__(self, *, origin: int, engine: sqlalchemy.engine.Engine,
314 namespace: Optional[str] = None):
315 self.origin = origin
316 self.namespace = namespace
317 self._engine = engine
318 self._session_connection: Optional[sqlalchemy.engine.Connection] = None
319 self._metadata: Optional[sqlalchemy.schema.MetaData] = None
320 self._tempTables: Set[str] = set()
322 def __repr__(self) -> str:
323 # Rather than try to reproduce all the parameters used to create
324 # the object, instead report the more useful information of the
325 # connection URL.
326 uri = str(self._engine.url)
327 if self.namespace:
328 uri += f"#{self.namespace}"
329 return f'{type(self).__name__}("{uri}")'
331 @classmethod
332 def makeDefaultUri(cls, root: str) -> Optional[str]:
333 """Create a default connection URI appropriate for the given root
334 directory, or `None` if there can be no such default.
335 """
336 return None
338 @classmethod
339 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None,
340 writeable: bool = True) -> Database:
341 """Construct a database from a SQLAlchemy URI.
343 Parameters
344 ----------
345 uri : `str`
346 A SQLAlchemy URI connection string.
347 origin : `int`
348 An integer ID that should be used as the default for any datasets,
349 quanta, or other entities that use a (autoincrement, origin)
350 compound primary key.
351 namespace : `str`, optional
352 A database namespace (i.e. schema) the new instance should be
353 associated with. If `None` (default), the namespace (if any) is
354 inferred from the URI.
355 writeable : `bool`, optional
356 If `True`, allow write operations on the database, including
357 ``CREATE TABLE``.
359 Returns
360 -------
361 db : `Database`
362 A new `Database` instance.
363 """
364 return cls.fromEngine(cls.makeEngine(uri, writeable=writeable),
365 origin=origin,
366 namespace=namespace,
367 writeable=writeable)
369 @classmethod
370 @abstractmethod
371 def makeEngine(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Engine:
372 """Create a `sqlalchemy.engine.Engine` from a SQLAlchemy URI.
374 Parameters
375 ----------
376 uri : `str`
377 A SQLAlchemy URI connection string.
378 writeable : `bool`, optional
379 If `True`, allow write operations on the database, including
380 ``CREATE TABLE``.
382 Returns
383 -------
384 engine : `sqlalchemy.engine.Engine`
385 A database engine.
387 Notes
388 -----
389 Subclasses that support other ways to connect to a database are
390 encouraged to add optional arguments to their implementation of this
391 method, as long as they maintain compatibility with the base class
392 call signature.
393 """
394 raise NotImplementedError()
396 @classmethod
397 @abstractmethod
398 def fromEngine(cls, engine: sqlalchemy.engine.Engine, *, origin: int,
399 namespace: Optional[str] = None, writeable: bool = True) -> Database:
400 """Create a new `Database` from an existing `sqlalchemy.engine.Engine`.
402 Parameters
403 ----------
404 engine : `sqllachemy.engine.Engine`
405 The engine for the database. May be shared between `Database`
406 instances.
407 origin : `int`
408 An integer ID that should be used as the default for any datasets,
409 quanta, or other entities that use a (autoincrement, origin)
410 compound primary key.
411 namespace : `str`, optional
412 A different database namespace (i.e. schema) the new instance
413 should be associated with. If `None` (default), the namespace
414 (if any) is inferred from the connection.
415 writeable : `bool`, optional
416 If `True`, allow write operations on the database, including
417 ``CREATE TABLE``.
419 Returns
420 -------
421 db : `Database`
422 A new `Database` instance.
424 Notes
425 -----
426 This method allows different `Database` instances to share the same
427 engine, which is desirable when they represent different namespaces
428 can be queried together.
429 """
430 raise NotImplementedError()
432 @contextmanager
433 def session(self) -> Iterator:
434 """Return a context manager that represents a session (persistent
435 connection to a database).
436 """
437 if self._session_connection is not None:
438 # session already started, just reuse that
439 yield Session(self)
440 else:
441 # open new connection and close it when done
442 self._session_connection = self._engine.connect()
443 yield Session(self)
444 self._session_connection.close()
445 self._session_connection = None
446 # Temporary tables only live within session
447 self._tempTables = set()
449 @contextmanager
450 def transaction(self, *, interrupting: bool = False, savepoint: bool = False,
451 lock: Iterable[sqlalchemy.schema.Table] = ()) -> Iterator:
452 """Return a context manager that represents a transaction.
454 Parameters
455 ----------
456 interrupting : `bool`, optional
457 If `True` (`False` is default), this transaction block may not be
458 nested without an outer one, and attempting to do so is a logic
459 (i.e. assertion) error.
460 savepoint : `bool`, optional
461 If `True` (`False` is default), create a `SAVEPOINT`, allowing
462 exceptions raised by the database (e.g. due to constraint
463 violations) during this transaction's context to be caught outside
464 it without also rolling back all operations in an outer transaction
465 block. If `False`, transactions may still be nested, but a
466 rollback may be generated at any level and affects all levels, and
467 commits are deferred until the outermost block completes. If any
468 outer transaction block was created with ``savepoint=True``, all
469 inner blocks will be as well (regardless of the actual value
470 passed). This has no effect if this is the outermost transaction.
471 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional
472 A list of tables to lock for the duration of this transaction.
473 These locks are guaranteed to prevent concurrent writes and allow
474 this transaction (only) to acquire the same locks (others should
475 block), but only prevent concurrent reads if the database engine
476 requires that in order to block concurrent writes.
478 Notes
479 -----
480 All transactions on a connection managed by one or more `Database`
481 instances _must_ go through this method, or transaction state will not
482 be correctly managed.
483 """
484 # need a connection, use session to manage it
485 with self.session():
486 assert self._session_connection is not None
487 connection = self._session_connection
488 assert not (interrupting and connection.in_transaction()), (
489 "Logic error in transaction nesting: an operation that would "
490 "interrupt the active transaction context has been requested."
491 )
492 # We remember whether we are already in a SAVEPOINT transaction via
493 # the connection object's 'info' dict, which is explicitly for user
494 # information like this. This is safer than a regular `Database`
495 # instance attribute, because it guards against multiple `Database`
496 # instances sharing the same connection. The need to use our own
497 # flag here to track whether we're in a nested transaction should
498 # go away in SQLAlchemy 1.4, which seems to have a
499 # `Connection.in_nested_transaction()` method.
500 savepoint = savepoint or connection.info.get(_IN_SAVEPOINT_TRANSACTION, False)
501 connection.info[_IN_SAVEPOINT_TRANSACTION] = savepoint
502 if connection.in_transaction() and savepoint:
503 trans = connection.begin_nested()
504 else:
505 # Use a regular (non-savepoint) transaction always for the
506 # outermost context, as well as when a savepoint was not
507 # requested.
508 trans = connection.begin()
509 self._lockTables(lock)
510 try:
511 yield
512 trans.commit()
513 except BaseException:
514 trans.rollback()
515 raise
516 finally:
517 if not connection.in_transaction():
518 connection.info.pop(_IN_SAVEPOINT_TRANSACTION, None)
520 @property
521 def _connection(self) -> sqlalchemy.engine.Connectable:
522 """Object that can be used to execute queries
523 (`sqlalchemy.engine.Connectable`)
524 """
525 return self._session_connection or self._engine
527 @abstractmethod
528 def _lockTables(self, tables: Iterable[sqlalchemy.schema.Table] = ()) -> None:
529 """Acquire locks on the given tables.
531 This is an implementation hook for subclasses, called by `transaction`.
532 It should not be called directly by other code.
534 Parameters
535 ----------
536 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional
537 A list of tables to lock for the duration of this transaction.
538 These locks are guaranteed to prevent concurrent writes and allow
539 this transaction (only) to acquire the same locks (others should
540 block), but only prevent concurrent reads if the database engine
541 requires that in order to block concurrent writes.
542 """
543 raise NotImplementedError()
545 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool:
546 """Check whether a table is writeable, either because the database
547 connection is read-write or the table is a temporary table.
549 Parameters
550 ----------
551 table : `sqlalchemy.schema.Table`
552 SQLAlchemy table object to check.
554 Returns
555 -------
556 writeable : `bool`
557 Whether this table is writeable.
558 """
559 return self.isWriteable() or table.key in self._tempTables
561 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None:
562 """Raise if the given table is not writeable, either because the
563 database connection is read-write or the table is a temporary table.
565 Parameters
566 ----------
567 table : `sqlalchemy.schema.Table`
568 SQLAlchemy table object to check.
569 msg : `str`, optional
570 If provided, raise `ReadOnlyDatabaseError` instead of returning
571 `False`, with this message.
572 """
573 if not self.isTableWriteable(table):
574 raise ReadOnlyDatabaseError(msg)
576 @contextmanager
577 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]:
578 """Return a context manager in which the database's static DDL schema
579 can be declared.
581 Parameters
582 ----------
583 create : `bool`
584 If `True`, attempt to create all tables at the end of the context.
585 If `False`, they will be assumed to already exist.
587 Returns
588 -------
589 schema : `StaticTablesContext`
590 A helper object that is used to add new tables.
592 Raises
593 ------
594 ReadOnlyDatabaseError
595 Raised if ``create`` is `True`, `Database.isWriteable` is `False`,
596 and one or more declared tables do not already exist.
598 Examples
599 --------
600 Given a `Database` instance ``db``::
602 with db.declareStaticTables(create=True) as schema:
603 schema.addTable("table1", TableSpec(...))
604 schema.addTable("table2", TableSpec(...))
606 Notes
607 -----
608 A database's static DDL schema must be declared before any dynamic
609 tables are managed via calls to `ensureTableExists` or
610 `getExistingTable`. The order in which static schema tables are added
611 inside the context block is unimportant; they will automatically be
612 sorted and added in an order consistent with their foreign key
613 relationships.
614 """
615 if create and not self.isWriteable():
616 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.")
617 self._metadata = sqlalchemy.MetaData(schema=self.namespace)
618 try:
619 context = StaticTablesContext(self)
620 if create and context._tableNames:
621 # Looks like database is already initalized, to avoid danger
622 # of modifying/destroying valid schema we refuse to do
623 # anything in this case
624 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.")
625 yield context
626 for table, foreignKey in context._foreignKeys:
627 table.append_constraint(foreignKey)
628 if create:
629 if self.namespace is not None:
630 if self.namespace not in context._inspector.get_schema_names():
631 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace))
632 # In our tables we have columns that make use of sqlalchemy
633 # Sequence objects. There is currently a bug in sqlalchemy that
634 # causes a deprecation warning to be thrown on a property of
635 # the Sequence object when the repr for the sequence is
636 # created. Here a filter is used to catch these deprecation
637 # warnings when tables are created.
638 with warnings.catch_warnings():
639 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning)
640 self._metadata.create_all(self._connection)
641 # call all initializer methods sequentially
642 for init in context._initializers:
643 init(self)
644 except BaseException:
645 self._metadata = None
646 raise
648 @abstractmethod
649 def isWriteable(self) -> bool:
650 """Return `True` if this database can be modified by this client.
651 """
652 raise NotImplementedError()
654 @abstractmethod
655 def __str__(self) -> str:
656 """Return a human-readable identifier for this `Database`, including
657 any namespace or schema that identifies its names within a `Registry`.
658 """
659 raise NotImplementedError()
661 @property
662 def dialect(self) -> sqlalchemy.engine.Dialect:
663 """The SQLAlchemy dialect for this database engine
664 (`sqlalchemy.engine.Dialect`).
665 """
666 return self._engine.dialect
668 def shrinkDatabaseEntityName(self, original: str) -> str:
669 """Return a version of the given name that fits within this database
670 engine's length limits for table, constraint, indexes, and sequence
671 names.
673 Implementations should not assume that simple truncation is safe,
674 because multiple long names often begin with the same prefix.
676 The default implementation simply returns the given name.
678 Parameters
679 ----------
680 original : `str`
681 The original name.
683 Returns
684 -------
685 shrunk : `str`
686 The new, possibly shortened name.
687 """
688 return original
690 def expandDatabaseEntityName(self, shrunk: str) -> str:
691 """Retrieve the original name for a database entity that was too long
692 to fit within the database engine's limits.
694 Parameters
695 ----------
696 original : `str`
697 The original name.
699 Returns
700 -------
701 shrunk : `str`
702 The new, possibly shortened name.
703 """
704 return shrunk
706 def _mangleTableName(self, name: str) -> str:
707 """Map a logical, user-visible table name to the true table name used
708 in the database.
710 The default implementation returns the given name unchanged.
712 Parameters
713 ----------
714 name : `str`
715 Input table name. Should not include a namespace (i.e. schema)
716 prefix.
718 Returns
719 -------
720 mangled : `str`
721 Mangled version of the table name (still with no namespace prefix).
723 Notes
724 -----
725 Reimplementations of this method must be idempotent - mangling an
726 already-mangled name must have no effect.
727 """
728 return name
730 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]:
731 """Create constraints based on this spec.
733 Parameters
734 ----------
735 table : `str`
736 Name of the table this column is being added to.
737 spec : `FieldSpec`
738 Specification for the field to be added.
740 Returns
741 -------
742 constraint : `list` of `sqlalchemy.CheckConstraint`
743 Constraint added for this column.
744 """
745 # By default we return no additional constraints
746 return []
748 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData,
749 **kwargs: Any) -> sqlalchemy.schema.Column:
750 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`.
752 Parameters
753 ----------
754 table : `str`
755 Name of the table this column is being added to.
756 spec : `FieldSpec`
757 Specification for the field to be added.
758 metadata : `sqlalchemy.MetaData`
759 SQLAlchemy representation of the DDL schema this field's table is
760 being added to.
761 **kwargs
762 Additional keyword arguments to forward to the
763 `sqlalchemy.schema.Column` constructor. This is provided to make
764 it easier for derived classes to delegate to ``super()`` while
765 making only minor changes.
767 Returns
768 -------
769 column : `sqlalchemy.schema.Column`
770 SQLAlchemy representation of the field.
771 """
772 args = [spec.name, spec.getSizedColumnType()]
773 if spec.autoincrement:
774 # Generate a sequence to use for auto incrementing for databases
775 # that do not support it natively. This will be ignored by
776 # sqlalchemy for databases that do support it.
777 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"),
778 metadata=metadata))
779 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}."
780 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey,
781 comment=spec.doc, server_default=spec.default, **kwargs)
783 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData,
784 **kwargs: Any) -> sqlalchemy.schema.ForeignKeyConstraint:
785 """Convert a `ForeignKeySpec` to a
786 `sqlalchemy.schema.ForeignKeyConstraint`.
788 Parameters
789 ----------
790 table : `str`
791 Name of the table this foreign key is being added to.
792 spec : `ForeignKeySpec`
793 Specification for the foreign key to be added.
794 metadata : `sqlalchemy.MetaData`
795 SQLAlchemy representation of the DDL schema this constraint is
796 being added to.
797 **kwargs
798 Additional keyword arguments to forward to the
799 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is
800 provided to make it easier for derived classes to delegate to
801 ``super()`` while making only minor changes.
803 Returns
804 -------
805 constraint : `sqlalchemy.schema.ForeignKeyConstraint`
806 SQLAlchemy representation of the constraint.
807 """
808 name = self.shrinkDatabaseEntityName(
809 "_".join(["fkey", table, self._mangleTableName(spec.table)]
810 + list(spec.target) + list(spec.source))
811 )
812 return sqlalchemy.schema.ForeignKeyConstraint(
813 spec.source,
814 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target],
815 name=name,
816 ondelete=spec.onDelete
817 )
819 def _convertExclusionConstraintSpec(self, table: str,
820 spec: Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...],
821 metadata: sqlalchemy.MetaData) -> sqlalchemy.schema.Constraint:
822 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy
823 constraint representation.
825 Parameters
826 ----------
827 table : `str`
828 Name of the table this constraint is being added to.
829 spec : `tuple` [ `str` or `type` ]
830 A tuple of `str` column names and the `type` object returned by
831 `getTimespanRepresentation` (which must appear exactly once),
832 indicating the order of the columns in the index used to back the
833 constraint.
834 metadata : `sqlalchemy.MetaData`
835 SQLAlchemy representation of the DDL schema this constraint is
836 being added to.
838 Returns
839 -------
840 constraint : `sqlalchemy.schema.Constraint`
841 SQLAlchemy representation of the constraint.
843 Raises
844 ------
845 NotImplementedError
846 Raised if this database does not support exclusion constraints.
847 """
848 raise NotImplementedError(f"Database {self} does not support exclusion constraints.")
850 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData,
851 **kwargs: Any) -> sqlalchemy.schema.Table:
852 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`.
854 Parameters
855 ----------
856 spec : `TableSpec`
857 Specification for the foreign key to be added.
858 metadata : `sqlalchemy.MetaData`
859 SQLAlchemy representation of the DDL schema this table is being
860 added to.
861 **kwargs
862 Additional keyword arguments to forward to the
863 `sqlalchemy.schema.Table` constructor. This is provided to make it
864 easier for derived classes to delegate to ``super()`` while making
865 only minor changes.
867 Returns
868 -------
869 table : `sqlalchemy.schema.Table`
870 SQLAlchemy representation of the table.
872 Notes
873 -----
874 This method does not handle ``spec.foreignKeys`` at all, in order to
875 avoid circular dependencies. These are added by higher-level logic in
876 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`.
877 """
878 name = self._mangleTableName(name)
879 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields]
881 # Add any column constraints
882 for fieldSpec in spec.fields:
883 args.extend(self._makeColumnConstraints(name, fieldSpec))
885 # Track indexes added for primary key and unique constraints, to make
886 # sure we don't add duplicate explicit or foreign key indexes for
887 # those.
888 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)}
889 args.extend(
890 sqlalchemy.schema.UniqueConstraint(
891 *columns,
892 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns)))
893 )
894 for columns in spec.unique
895 )
896 allIndexes.update(spec.unique)
897 args.extend(
898 sqlalchemy.schema.Index(
899 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))),
900 *columns,
901 unique=(columns in spec.unique)
902 )
903 for columns in spec.indexes if columns not in allIndexes
904 )
905 allIndexes.update(spec.indexes)
906 args.extend(
907 sqlalchemy.schema.Index(
908 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)),
909 *fk.source,
910 )
911 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes
912 )
914 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion)
916 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}."
917 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwargs)
919 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
920 """Ensure that a table with the given name and specification exists,
921 creating it if necessary.
923 Parameters
924 ----------
925 name : `str`
926 Name of the table (not including namespace qualifiers).
927 spec : `TableSpec`
928 Specification for the table. This will be used when creating the
929 table, and *may* be used when obtaining an existing table to check
930 for consistency, but no such check is guaranteed.
932 Returns
933 -------
934 table : `sqlalchemy.schema.Table`
935 SQLAlchemy representation of the table.
937 Raises
938 ------
939 ReadOnlyDatabaseError
940 Raised if `isWriteable` returns `False`, and the table does not
941 already exist.
942 DatabaseConflictError
943 Raised if the table exists but ``spec`` is inconsistent with its
944 definition.
946 Notes
947 -----
948 This method may not be called within transactions. It may be called on
949 read-only databases if and only if the table does in fact already
950 exist.
952 Subclasses may override this method, but usually should not need to.
953 """
954 # TODO: if _engine is used to make a table then it uses separate
955 # connection and should not interfere with current transaction
956 assert self._session_connection is None or not self._session_connection.in_transaction(), \
957 "Table creation interrupts transactions."
958 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
959 table = self.getExistingTable(name, spec)
960 if table is not None:
961 return table
962 if not self.isWriteable():
963 raise ReadOnlyDatabaseError(
964 f"Table {name} does not exist, and cannot be created "
965 f"because database {self} is read-only."
966 )
967 table = self._convertTableSpec(name, spec, self._metadata)
968 for foreignKeySpec in spec.foreignKeys:
969 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
970 table.create(self._connection)
971 return table
973 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]:
974 """Obtain an existing table with the given name and specification.
976 Parameters
977 ----------
978 name : `str`
979 Name of the table (not including namespace qualifiers).
980 spec : `TableSpec`
981 Specification for the table. This will be used when creating the
982 SQLAlchemy representation of the table, and it is used to
983 check that the actual table in the database is consistent.
985 Returns
986 -------
987 table : `sqlalchemy.schema.Table` or `None`
988 SQLAlchemy representation of the table, or `None` if it does not
989 exist.
991 Raises
992 ------
993 DatabaseConflictError
994 Raised if the table exists but ``spec`` is inconsistent with its
995 definition.
997 Notes
998 -----
999 This method can be called within transactions and never modifies the
1000 database.
1002 Subclasses may override this method, but usually should not need to.
1003 """
1004 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
1005 name = self._mangleTableName(name)
1006 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}")
1007 if table is not None:
1008 if spec.fields.names != set(table.columns.keys()):
1009 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new "
1010 f"specification has columns {list(spec.fields.names)}, while "
1011 f"the previous definition has {list(table.columns.keys())}.")
1012 else:
1013 inspector = sqlalchemy.inspect(self._connection)
1014 if name in inspector.get_table_names(schema=self.namespace):
1015 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace))
1016 table = self._convertTableSpec(name, spec, self._metadata)
1017 for foreignKeySpec in spec.foreignKeys:
1018 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
1019 return table
1020 return table
1022 @classmethod
1023 def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]:
1024 """Return a `type` that encapsulates the way `Timespan` objects are
1025 stored in this database.
1027 `Database` does not automatically use the return type of this method
1028 anywhere else; calling code is responsible for making sure that DDL
1029 and queries are consistent with it.
1031 Returns
1032 -------
1033 TimespanReprClass : `type` (`TimespanDatabaseRepresention` subclass)
1034 A type that encapsulates the way `Timespan` objects should be
1035 stored in this database.
1037 Notes
1038 -----
1039 There are two big reasons we've decided to keep timespan-mangling logic
1040 outside the `Database` implementations, even though the choice of
1041 representation is ultimately up to a `Database` implementation:
1043 - Timespans appear in relatively few tables and queries in our
1044 typical usage, and the code that operates on them is already aware
1045 that it is working with timespans. In contrast, a
1046 timespan-representation-aware implementation of, say, `insert`,
1047 would need to have extra logic to identify when timespan-mangling
1048 needed to occur, which would usually be useless overhead.
1050 - SQLAlchemy's rich SELECT query expression system has no way to wrap
1051 multiple columns in a single expression object (the ORM does, but
1052 we are not using the ORM). So we would have to wrap _much_ more of
1053 that code in our own interfaces to encapsulate timespan
1054 representations there.
1055 """
1056 return TimespanDatabaseRepresentation.Compound
1058 @classmethod
1059 def getSpatialRegionRepresentation(cls) -> Type[SpatialRegionDatabaseRepresentation]:
1060 """Return a `type` that encapsulates the way `lsst.sphgeom.Region`
1061 objects are stored in this database.
1063 `Database` does not automatically use the return type of this method
1064 anywhere else; calling code is responsible for making sure that DDL
1065 and queries are consistent with it.
1067 Returns
1068 -------
1069 RegionReprClass : `type` (`SpatialRegionDatabaseRepresention` subclass)
1070 A type that encapsulates the way `lsst.sphgeom.Region` objects
1071 should be stored in this database.
1073 Notes
1074 -----
1075 See `getTimespanRepresentation` for comments on why this method is not
1076 more tightly integrated with the rest of the `Database` interface.
1077 """
1078 return SpatialRegionDatabaseRepresentation
1080 def sync(self, table: sqlalchemy.schema.Table, *,
1081 keys: Dict[str, Any],
1082 compared: Optional[Dict[str, Any]] = None,
1083 extra: Optional[Dict[str, Any]] = None,
1084 returning: Optional[Sequence[str]] = None,
1085 ) -> Tuple[Optional[Dict[str, Any]], bool]:
1086 """Insert into a table as necessary to ensure database contains
1087 values equivalent to the given ones.
1089 Parameters
1090 ----------
1091 table : `sqlalchemy.schema.Table`
1092 Table to be queried and possibly inserted into.
1093 keys : `dict`
1094 Column name-value pairs used to search for an existing row; must
1095 be a combination that can be used to select a single row if one
1096 exists. If such a row does not exist, these values are used in
1097 the insert.
1098 compared : `dict`, optional
1099 Column name-value pairs that are compared to those in any existing
1100 row. If such a row does not exist, these rows are used in the
1101 insert.
1102 extra : `dict`, optional
1103 Column name-value pairs that are ignored if a matching row exists,
1104 but used in an insert if one is necessary.
1105 returning : `~collections.abc.Sequence` of `str`, optional
1106 The names of columns whose values should be returned.
1108 Returns
1109 -------
1110 row : `dict`, optional
1111 The value of the fields indicated by ``returning``, or `None` if
1112 ``returning`` is `None`.
1113 inserted : `bool`
1114 If `True`, a new row was inserted.
1116 Raises
1117 ------
1118 DatabaseConflictError
1119 Raised if the values in ``compared`` do not match the values in the
1120 database.
1121 ReadOnlyDatabaseError
1122 Raised if `isWriteable` returns `False`, and no matching record
1123 already exists.
1125 Notes
1126 -----
1127 May be used inside transaction contexts, so implementations may not
1128 perform operations that interrupt transactions.
1130 It may be called on read-only databases if and only if the matching row
1131 does in fact already exist.
1132 """
1134 def check() -> Tuple[int, Optional[List[str]], Optional[List]]:
1135 """Query for a row that matches the ``key`` argument, and compare
1136 to what was given by the caller.
1138 Returns
1139 -------
1140 n : `int`
1141 Number of matching rows. ``n != 1`` is always an error, but
1142 it's a different kind of error depending on where `check` is
1143 being called.
1144 bad : `list` of `str`, or `None`
1145 The subset of the keys of ``compared`` for which the existing
1146 values did not match the given one. Once again, ``not bad``
1147 is always an error, but a different kind on context. `None`
1148 if ``n != 1``
1149 result : `list` or `None`
1150 Results in the database that correspond to the columns given
1151 in ``returning``, or `None` if ``returning is None``.
1152 """
1153 toSelect: Set[str] = set()
1154 if compared is not None:
1155 toSelect.update(compared.keys())
1156 if returning is not None:
1157 toSelect.update(returning)
1158 if not toSelect:
1159 # Need to select some column, even if we just want to see
1160 # how many rows we get back.
1161 toSelect.add(next(iter(keys.keys())))
1162 selectSql = sqlalchemy.sql.select(
1163 [table.columns[k].label(k) for k in toSelect]
1164 ).select_from(table).where(
1165 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])
1166 )
1167 fetched = list(self._connection.execute(selectSql).fetchall())
1168 if len(fetched) != 1:
1169 return len(fetched), None, None
1170 existing = fetched[0]
1171 if compared is not None:
1173 def safeNotEqual(a: Any, b: Any) -> bool:
1174 if isinstance(a, astropy.time.Time):
1175 return not time_utils.TimeConverter().times_equal(a, b)
1176 return a != b
1178 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}"
1179 for k, v in compared.items()
1180 if safeNotEqual(existing[k], v)]
1181 else:
1182 inconsistencies = []
1183 if returning is not None:
1184 toReturn: Optional[list] = [existing[k] for k in returning]
1185 else:
1186 toReturn = None
1187 return 1, inconsistencies, toReturn
1189 if self.isTableWriteable(table):
1190 # Try an insert first, but allow it to fail (in only specific
1191 # ways).
1192 row = keys.copy()
1193 if compared is not None:
1194 row.update(compared)
1195 if extra is not None:
1196 row.update(extra)
1197 with self.transaction(lock=[table]):
1198 inserted = bool(self.ensure(table, row))
1199 # Need to perform check() for this branch inside the
1200 # transaction, so we roll back an insert that didn't do
1201 # what we expected. That limits the extent to which we
1202 # can reduce duplication between this block and the other
1203 # ones that perform similar logic.
1204 n, bad, result = check()
1205 if n < 1:
1206 raise ConflictingDefinitionError(
1207 f"Attempted to ensure {row} exists by inserting it with ON CONFLICT IGNORE, "
1208 f"but a post-insert query on {keys} returned no results. "
1209 f"Insert was {'' if inserted else 'not '}reported as successful. "
1210 "This can occur if the insert violated a database constraint other than the "
1211 "unique constraint or primary key used to identify the row in this call."
1212 )
1213 elif n > 1:
1214 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
1215 f"unique constraint for table {table.name}.")
1216 elif bad:
1217 if inserted:
1218 raise RuntimeError(
1219 f"Conflict ({bad}) in sync after successful insert; this is "
1220 "possible if the same table is being updated by a concurrent "
1221 "process that isn't using sync, but it may also be a bug in "
1222 "daf_butler."
1223 )
1224 else:
1225 raise DatabaseConflictError(
1226 f"Conflict in sync for table {table.name} on column(s) {bad}."
1227 )
1228 else:
1229 # Database is not writeable; just see if the row exists.
1230 n, bad, result = check()
1231 if n < 1:
1232 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.")
1233 elif n > 1:
1234 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.")
1235 elif bad:
1236 raise DatabaseConflictError(
1237 f"Conflict in sync for table {table.name} on column(s) {bad}."
1238 )
1239 inserted = False
1240 if returning is None:
1241 return None, inserted
1242 else:
1243 assert result is not None
1244 return {k: v for k, v in zip(returning, result)}, inserted
1246 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False,
1247 select: Optional[sqlalchemy.sql.Select] = None,
1248 names: Optional[Iterable[str]] = None,
1249 ) -> Optional[List[int]]:
1250 """Insert one or more rows into a table, optionally returning
1251 autoincrement primary key values.
1253 Parameters
1254 ----------
1255 table : `sqlalchemy.schema.Table`
1256 Table rows should be inserted into.
1257 returnIds: `bool`
1258 If `True` (`False` is default), return the values of the table's
1259 autoincrement primary key field (which much exist).
1260 select : `sqlalchemy.sql.Select`, optional
1261 A SELECT query expression to insert rows from. Cannot be provided
1262 with either ``rows`` or ``returnIds=True``.
1263 names : `Iterable` [ `str` ], optional
1264 Names of columns in ``table`` to be populated, ordered to match the
1265 columns returned by ``select``. Ignored if ``select`` is `None`.
1266 If not provided, the columns returned by ``select`` must be named
1267 to match the desired columns of ``table``.
1268 *rows
1269 Positional arguments are the rows to be inserted, as dictionaries
1270 mapping column name to value. The keys in all dictionaries must
1271 be the same.
1273 Returns
1274 -------
1275 ids : `None`, or `list` of `int`
1276 If ``returnIds`` is `True`, a `list` containing the inserted
1277 values for the table's autoincrement primary key.
1279 Raises
1280 ------
1281 ReadOnlyDatabaseError
1282 Raised if `isWriteable` returns `False` when this method is called.
1284 Notes
1285 -----
1286 The default implementation uses bulk insert syntax when ``returnIds``
1287 is `False`, and a loop over single-row insert operations when it is
1288 `True`.
1290 Derived classes should reimplement when they can provide a more
1291 efficient implementation (especially for the latter case).
1293 May be used inside transaction contexts, so implementations may not
1294 perform operations that interrupt transactions.
1295 """
1296 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.")
1297 if select is not None and (rows or returnIds):
1298 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.")
1299 if not rows and select is None:
1300 if returnIds:
1301 return []
1302 else:
1303 return None
1304 if not returnIds:
1305 if select is not None:
1306 if names is None:
1307 # columns() is deprecated since 1.4, but
1308 # selected_columns() method did not exist in 1.3.
1309 if hasattr(select, "selected_columns"):
1310 names = select.selected_columns.keys()
1311 else:
1312 names = select.columns.keys()
1313 self._connection.execute(table.insert().from_select(names, select))
1314 else:
1315 self._connection.execute(table.insert(), *rows)
1316 return None
1317 else:
1318 sql = table.insert()
1319 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows]
1321 @abstractmethod
1322 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None:
1323 """Insert one or more rows into a table, replacing any existing rows
1324 for which insertion of a new row would violate the primary key
1325 constraint.
1327 Parameters
1328 ----------
1329 table : `sqlalchemy.schema.Table`
1330 Table rows should be inserted into.
1331 *rows
1332 Positional arguments are the rows to be inserted, as dictionaries
1333 mapping column name to value. The keys in all dictionaries must
1334 be the same.
1336 Raises
1337 ------
1338 ReadOnlyDatabaseError
1339 Raised if `isWriteable` returns `False` when this method is called.
1341 Notes
1342 -----
1343 May be used inside transaction contexts, so implementations may not
1344 perform operations that interrupt transactions.
1346 Implementations should raise a `sqlalchemy.exc.IntegrityError`
1347 exception when a constraint other than the primary key would be
1348 violated.
1350 Implementations are not required to support `replace` on tables
1351 with autoincrement keys.
1352 """
1353 raise NotImplementedError()
1355 @abstractmethod
1356 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict) -> int:
1357 """Insert one or more rows into a table, skipping any rows for which
1358 insertion would violate any constraint.
1360 Parameters
1361 ----------
1362 table : `sqlalchemy.schema.Table`
1363 Table rows should be inserted into.
1364 *rows
1365 Positional arguments are the rows to be inserted, as dictionaries
1366 mapping column name to value. The keys in all dictionaries must
1367 be the same.
1369 Returns
1370 -------
1371 count : `int`
1372 The number of rows actually inserted.
1374 Raises
1375 ------
1376 ReadOnlyDatabaseError
1377 Raised if `isWriteable` returns `False` when this method is called.
1378 This is raised even if the operation would do nothing even on a
1379 writeable database.
1381 Notes
1382 -----
1383 May be used inside transaction contexts, so implementations may not
1384 perform operations that interrupt transactions.
1386 Implementations are not required to support `ensure` on tables
1387 with autoincrement keys.
1388 """
1389 raise NotImplementedError()
1391 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int:
1392 """Delete one or more rows from a table.
1394 Parameters
1395 ----------
1396 table : `sqlalchemy.schema.Table`
1397 Table that rows should be deleted from.
1398 columns: `~collections.abc.Iterable` of `str`
1399 The names of columns that will be used to constrain the rows to
1400 be deleted; these will be combined via ``AND`` to form the
1401 ``WHERE`` clause of the delete query.
1402 *rows
1403 Positional arguments are the keys of rows to be deleted, as
1404 dictionaries mapping column name to value. The keys in all
1405 dictionaries must be exactly the names in ``columns``.
1407 Returns
1408 -------
1409 count : `int`
1410 Number of rows deleted.
1412 Raises
1413 ------
1414 ReadOnlyDatabaseError
1415 Raised if `isWriteable` returns `False` when this method is called.
1417 Notes
1418 -----
1419 May be used inside transaction contexts, so implementations may not
1420 perform operations that interrupt transactions.
1422 The default implementation should be sufficient for most derived
1423 classes.
1424 """
1425 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.")
1426 if columns and not rows:
1427 # If there are no columns, this operation is supposed to delete
1428 # everything (so we proceed as usual). But if there are columns,
1429 # but no rows, it was a constrained bulk operation where the
1430 # constraint is that no rows match, and we should short-circuit
1431 # while reporting that no rows were affected.
1432 return 0
1433 sql = table.delete()
1434 columns = list(columns) # Force iterators to list
1436 # More efficient to use IN operator if there is only one
1437 # variable changing across all rows.
1438 content: Dict[str, Set] = defaultdict(set)
1439 if len(columns) == 1:
1440 # Nothing to calculate since we can always use IN
1441 column = columns[0]
1442 changing_columns = [column]
1443 content[column] = set(row[column] for row in rows)
1444 else:
1445 for row in rows:
1446 for k, v in row.items():
1447 content[k].add(v)
1448 changing_columns = [col for col, values in content.items() if len(values) > 1]
1450 if len(changing_columns) != 1:
1451 # More than one column changes each time so do explicit bind
1452 # parameters and have each row processed separately.
1453 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns]
1454 if whereTerms:
1455 sql = sql.where(sqlalchemy.sql.and_(*whereTerms))
1456 return self._connection.execute(sql, *rows).rowcount
1457 else:
1458 # One of the columns has changing values but any others are
1459 # fixed. In this case we can use an IN operator and be more
1460 # efficient.
1461 name = changing_columns.pop()
1463 # Simple where clause for the unchanging columns
1464 clauses = []
1465 for k, v in content.items():
1466 if k == name:
1467 continue
1468 column = table.columns[k]
1469 # The set only has one element
1470 clauses.append(column == v.pop())
1472 # The IN operator will not work for "infinite" numbers of
1473 # rows so must batch it up into distinct calls.
1474 in_content = list(content[name])
1475 n_elements = len(in_content)
1477 rowcount = 0
1478 iposn = 0
1479 n_per_loop = 1_000 # Controls how many items to put in IN clause
1480 for iposn in range(0, n_elements, n_per_loop):
1481 endpos = iposn + n_per_loop
1482 in_clause = table.columns[name].in_(in_content[iposn:endpos])
1484 newsql = sql.where(sqlalchemy.sql.and_(*clauses, in_clause))
1485 rowcount += self._connection.execute(newsql).rowcount
1486 return rowcount
1488 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int:
1489 """Update one or more rows in a table.
1491 Parameters
1492 ----------
1493 table : `sqlalchemy.schema.Table`
1494 Table containing the rows to be updated.
1495 where : `dict` [`str`, `str`]
1496 A mapping from the names of columns that will be used to search for
1497 existing rows to the keys that will hold these values in the
1498 ``rows`` dictionaries. Note that these may not be the same due to
1499 SQLAlchemy limitations.
1500 *rows
1501 Positional arguments are the rows to be updated. The keys in all
1502 dictionaries must be the same, and may correspond to either a
1503 value in the ``where`` dictionary or the name of a column to be
1504 updated.
1506 Returns
1507 -------
1508 count : `int`
1509 Number of rows matched (regardless of whether the update actually
1510 modified them).
1512 Raises
1513 ------
1514 ReadOnlyDatabaseError
1515 Raised if `isWriteable` returns `False` when this method is called.
1517 Notes
1518 -----
1519 May be used inside transaction contexts, so implementations may not
1520 perform operations that interrupt transactions.
1522 The default implementation should be sufficient for most derived
1523 classes.
1524 """
1525 self.assertTableWriteable(table, f"Cannot update read-only table {table}.")
1526 if not rows:
1527 return 0
1528 sql = table.update().where(
1529 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()])
1530 )
1531 return self._connection.execute(sql, *rows).rowcount
1533 def query(self, sql: sqlalchemy.sql.FromClause,
1534 *args: Any, **kwargs: Any) -> sqlalchemy.engine.ResultProxy:
1535 """Run a SELECT query against the database.
1537 Parameters
1538 ----------
1539 sql : `sqlalchemy.sql.FromClause`
1540 A SQLAlchemy representation of a ``SELECT`` query.
1541 *args
1542 Additional positional arguments are forwarded to
1543 `sqlalchemy.engine.Connection.execute`.
1544 **kwargs
1545 Additional keyword arguments are forwarded to
1546 `sqlalchemy.engine.Connection.execute`.
1548 Returns
1549 -------
1550 result : `sqlalchemy.engine.ResultProxy`
1551 Query results.
1553 Notes
1554 -----
1555 The default implementation should be sufficient for most derived
1556 classes.
1557 """
1558 # TODO: should we guard against non-SELECT queries here?
1559 return self._connection.execute(sql, *args, **kwargs)
1561 origin: int
1562 """An integer ID that should be used as the default for any datasets,
1563 quanta, or other entities that use a (autoincrement, origin) compound
1564 primary key (`int`).
1565 """
1567 namespace: Optional[str]
1568 """The schema or namespace this database instance is associated with
1569 (`str` or `None`).
1570 """