Coverage for python/lsst/daf/butler/registry/interfaces/_database.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = [
24 "Database",
25 "ReadOnlyDatabaseError",
26 "DatabaseConflictError",
27 "StaticTablesContext",
28]
30from abc import ABC, abstractmethod
31from contextlib import contextmanager
32from typing import (
33 Any,
34 Dict,
35 Iterable,
36 List,
37 Optional,
38 Sequence,
39 Tuple,
40)
41import warnings
43import astropy.time
44import sqlalchemy
46from ...core import ddl, time_utils
49def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: Dict[str, Any]):
50 """Test that the definition of a table in a `ddl.TableSpec` and from
51 database introspection are consistent.
53 Parameters
54 ----------
55 name : `str`
56 Name of the table (only used in error messages).
57 spec : `ddl.TableSpec`
58 Specification of the table.
59 inspection : `dict`
60 Dictionary returned by
61 `sqlalchemy.engine.reflection.Inspector.get_columns`.
63 Raises
64 ------
65 DatabaseConflictError
66 Raised if the definitions are inconsistent.
67 """
68 columnNames = [c["name"] for c in inspection]
69 if spec.fields.names != set(columnNames):
70 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; "
71 f"specification has columns {list(spec.fields.names)}, while the "
72 f"table in the database has {columnNames}.")
75class ReadOnlyDatabaseError(RuntimeError):
76 """Exception raised when a write operation is called on a read-only
77 `Database`.
78 """
81class DatabaseConflictError(RuntimeError):
82 """Exception raised when database content (row values or schema entities)
83 are inconsistent with what this client expects.
84 """
87class StaticTablesContext:
88 """Helper class used to declare the static schema for a registry layer
89 in a database.
91 An instance of this class is returned by `Database.declareStaticTables`,
92 which should be the only way it should be constructed.
93 """
95 def __init__(self, db: Database):
96 self._db = db
97 self._foreignKeys = []
98 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection)
99 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace))
101 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table:
102 """Add a new table to the schema, returning its sqlalchemy
103 representation.
105 The new table may not actually be created until the end of the
106 context created by `Database.declareStaticTables`, allowing tables
107 to be declared in any order even in the presence of foreign key
108 relationships.
109 """
110 name = self._db._mangleTableName(name)
111 if name in self._tableNames:
112 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name,
113 schema=self._db.namespace))
114 table = self._db._convertTableSpec(name, spec, self._db._metadata)
115 for foreignKeySpec in spec.foreignKeys:
116 self._foreignKeys.append(
117 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata))
118 )
119 return table
121 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]:
122 """Add a named tuple of tables to the schema, returning their
123 SQLAlchemy representations in a named tuple of the same type.
125 The new tables may not actually be created until the end of the
126 context created by `Database.declareStaticTables`, allowing tables
127 to be declared in any order even in the presence of foreign key
128 relationships.
130 Notes
131 -----
132 ``specs`` *must* be an instance of a type created by
133 `collections.namedtuple`, not just regular tuple, and the returned
134 object is guaranteed to be the same. Because `~collections.namedtuple`
135 is just a factory for `type` objects, not an actual type itself,
136 we cannot represent this with type annotations.
137 """
138 return specs._make(self.addTable(name, spec) for name, spec in zip(specs._fields, specs))
141class Database(ABC):
142 """An abstract interface that represents a particular database engine's
143 representation of a single schema/namespace/database.
145 Parameters
146 ----------
147 origin : `int`
148 An integer ID that should be used as the default for any datasets,
149 quanta, or other entities that use a (autoincrement, origin) compound
150 primary key.
151 connection : `sqlalchemy.engine.Connection`
152 The SQLAlchemy connection this `Database` wraps.
153 namespace : `str`, optional
154 Name of the schema or namespace this instance is associated with.
155 This is passed as the ``schema`` argument when constructing a
156 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to
157 avoid confusion between "schema means namespace" and "schema means
158 table definitions".
160 Notes
161 -----
162 `Database` requires all write operations to go through its special named
163 methods. Our write patterns are sufficiently simple that we don't really
164 need the full flexibility of SQL insert/update/delete syntax, and we need
165 non-standard (but common) functionality in these operations sufficiently
166 often that it seems worthwhile to provide our own generic API.
168 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via
169 their SQLAlchemy representation) to be run, as we expect these to require
170 significantly more sophistication while still being limited to standard
171 SQL.
173 `Database` itself has several underscore-prefixed attributes:
175 - ``_cs``: SQLAlchemy objects representing the connection and transaction
176 state.
177 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing
178 the tables and other schema entities.
180 These are considered protected (derived classes may access them, but other
181 code should not), and read-only, aside from executing SQL via
182 ``_connection``.
183 """
185 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection,
186 namespace: Optional[str] = None):
187 self.origin = origin
188 self.namespace = namespace
189 self._connection = connection
190 self._metadata = None
192 @classmethod
193 def makeDefaultUri(cls, root: str) -> Optional[str]:
194 """Create a default connection URI appropriate for the given root
195 directory, or `None` if there can be no such default.
196 """
197 return None
199 @classmethod
200 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None,
201 writeable: bool = True) -> Database:
202 """Construct a database from a SQLAlchemy URI.
204 Parameters
205 ----------
206 uri : `str`
207 A SQLAlchemy URI connection string.
208 origin : `int`
209 An integer ID that should be used as the default for any datasets,
210 quanta, or other entities that use a (autoincrement, origin)
211 compound primary key.
212 namespace : `str`, optional
213 A database namespace (i.e. schema) the new instance should be
214 associated with. If `None` (default), the namespace (if any) is
215 inferred from the URI.
216 writeable : `bool`, optional
217 If `True`, allow write operations on the database, including
218 ``CREATE TABLE``.
220 Returns
221 -------
222 db : `Database`
223 A new `Database` instance.
224 """
225 return cls.fromConnection(cls.connect(uri, writeable=writeable),
226 origin=origin,
227 namespace=namespace,
228 writeable=writeable)
230 @classmethod
231 @abstractmethod
232 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection:
233 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI.
235 Parameters
236 ----------
237 uri : `str`
238 A SQLAlchemy URI connection string.
239 origin : `int`
240 An integer ID that should be used as the default for any datasets,
241 quanta, or other entities that use a (autoincrement, origin)
242 compound primary key.
243 writeable : `bool`, optional
244 If `True`, allow write operations on the database, including
245 ``CREATE TABLE``.
247 Returns
248 -------
249 connection : `sqlalchemy.engine.Connection`
250 A database connection.
252 Notes
253 -----
254 Subclasses that support other ways to connect to a database are
255 encouraged to add optional arguments to their implementation of this
256 method, as long as they maintain compatibility with the base class
257 call signature.
258 """
259 raise NotImplementedError()
261 @classmethod
262 @abstractmethod
263 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int,
264 namespace: Optional[str] = None, writeable: bool = True) -> Database:
265 """Create a new `Database` from an existing
266 `sqlalchemy.engine.Connection`.
268 Parameters
269 ----------
270 connection : `sqllachemy.engine.Connection`
271 The connection for the the database. May be shared between
272 `Database` instances.
273 origin : `int`
274 An integer ID that should be used as the default for any datasets,
275 quanta, or other entities that use a (autoincrement, origin)
276 compound primary key.
277 namespace : `str`, optional
278 A different database namespace (i.e. schema) the new instance
279 should be associated with. If `None` (default), the namespace
280 (if any) is inferred from the connection.
281 writeable : `bool`, optional
282 If `True`, allow write operations on the database, including
283 ``CREATE TABLE``.
285 Returns
286 -------
287 db : `Database`
288 A new `Database` instance.
290 Notes
291 -----
292 This method allows different `Database` instances to share the same
293 connection, which is desirable when they represent different namespaces
294 can be queried together. This also ties their transaction state,
295 however; starting a transaction in any database automatically starts
296 on in all other databases.
297 """
298 raise NotImplementedError()
300 @contextmanager
301 def transaction(self, *, interrupting: bool = False) -> None:
302 """Return a context manager that represents a transaction.
304 Parameters
305 ----------
306 interrupting : `bool`
307 If `True`, this transaction block needs to be able to interrupt
308 any existing one in order to yield correct behavior.
309 """
310 assert not (interrupting and self._connection.in_transaction()), (
311 "Logic error in transaction nesting: an operation that would "
312 "interrupt the active transaction context has been requested."
313 )
314 if self._connection.in_transaction():
315 trans = self._connection.begin_nested()
316 else:
317 # Use a regular (non-savepoint) transaction only for the outermost
318 # context.
319 trans = self._connection.begin()
320 try:
321 yield
322 trans.commit()
323 except BaseException:
324 trans.rollback()
325 raise
327 @contextmanager
328 def declareStaticTables(self, *, create: bool) -> StaticTablesContext:
329 """Return a context manager in which the database's static DDL schema
330 can be declared.
332 Parameters
333 ----------
334 create : `bool`
335 If `True`, attempt to create all tables at the end of the context.
336 If `False`, they will be assumed to already exist.
338 Returns
339 -------
340 schema : `StaticTablesContext`
341 A helper object that is used to add new tables.
343 Raises
344 ------
345 ReadOnlyDatabaseError
346 Raised if ``create`` is `True`, `Database.isWriteable` is `False`,
347 and one or more declared tables do not already exist.
349 Examples
350 --------
351 Given a `Database` instance ``db``::
353 with db.declareStaticTables(create=True) as schema:
354 schema.addTable("table1", TableSpec(...))
355 schema.addTable("table2", TableSpec(...))
357 Notes
358 -----
359 A database's static DDL schema must be declared before any dynamic
360 tables are managed via calls to `ensureTableExists` or
361 `getExistingTable`. The order in which static schema tables are added
362 inside the context block is unimportant; they will automatically be
363 sorted and added in an order consistent with their foreign key
364 relationships.
365 """
366 if create and not self.isWriteable():
367 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.")
368 self._metadata = sqlalchemy.MetaData(schema=self.namespace)
369 try:
370 context = StaticTablesContext(self)
371 yield context
372 for table, foreignKey in context._foreignKeys:
373 table.append_constraint(foreignKey)
374 if create:
375 if self.namespace is not None:
376 if self.namespace not in context._inspector.get_schema_names():
377 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace))
378 # In our tables we have columns that make use of sqlalchemy
379 # Sequence objects. There is currently a bug in sqlalchmey that
380 # causes a deprecation warning to be thrown on a property of
381 # the Sequence object when the repr for the sequence is
382 # created. Here a filter is used to catch these deprecation
383 # warnings when tables are created.
384 with warnings.catch_warnings():
385 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning)
386 self._metadata.create_all(self._connection)
387 except BaseException:
388 self._metadata.drop_all(self._connection)
389 self._metadata = None
390 raise
392 @abstractmethod
393 def isWriteable(self) -> bool:
394 """Return `True` if this database can be modified by this client.
395 """
396 raise NotImplementedError()
398 @abstractmethod
399 def __str__(self) -> str:
400 """Return a human-readable identifier for this `Database`, including
401 any namespace or schema that identifies its names within a `Registry`.
402 """
403 raise NotImplementedError()
405 def shrinkDatabaseEntityName(self, original: str) -> str:
406 """Return a version of the given name that fits within this database
407 engine's length limits for table, constraint, indexes, and sequence
408 names.
410 Implementations should not assume that simple truncation is safe,
411 because multiple long names often begin with the same prefix.
413 The default implementation simply returns the given name.
415 Parameters
416 ----------
417 original : `str`
418 The original name.
420 Returns
421 -------
422 shrunk : `str`
423 The new, possibly shortened name.
424 """
425 return original
427 def expandDatabaseEntityName(self, shrunk: str) -> str:
428 """Retrieve the original name for a database entity that was too long
429 to fit within the database engine's limits.
431 Parameters
432 ----------
433 original : `str`
434 The original name.
436 Returns
437 -------
438 shrunk : `str`
439 The new, possibly shortened name.
440 """
441 return shrunk
443 def _mangleTableName(self, name: str) -> str:
444 """Map a logical, user-visible table name to the true table name used
445 in the database.
447 The default implementation returns the given name unchanged.
449 Parameters
450 ----------
451 name : `str`
452 Input table name. Should not include a namespace (i.e. schema)
453 prefix.
455 Returns
456 -------
457 mangled : `str`
458 Mangled version of the table name (still with no namespace prefix).
460 Notes
461 -----
462 Reimplementations of this method must be idempotent - mangling an
463 already-mangled name must have no effect.
464 """
465 return name
467 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData,
468 **kwds) -> sqlalchemy.schema.Column:
469 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`.
471 Parameters
472 ----------
473 table : `str`
474 Name of the table this column is being added to.
475 spec : `FieldSpec`
476 Specification for the field to be added.
477 metadata : `sqlalchemy.MetaData`
478 SQLAlchemy representation of the DDL schema this field's table is
479 being added to.
480 **kwds
481 Additional keyword arguments to forward to the
482 `sqlalchemy.schema.Column` constructor. This is provided to make
483 it easier for derived classes to delegate to ``super()`` while
484 making only minor changes.
486 Returns
487 -------
488 column : `sqlalchemy.schema.Column`
489 SQLAlchemy representation of the field.
490 """
491 args = [spec.name, spec.getSizedColumnType()]
492 if spec.autoincrement:
493 # Generate a sequence to use for auto incrementing for databases
494 # that do not support it natively. This will be ignored by
495 # sqlalchemy for databases that do support it.
496 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"),
497 metadata=metadata))
498 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}."
499 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey,
500 comment=spec.doc, **kwds)
502 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData,
503 **kwds) -> sqlalchemy.schema.ForeignKeyConstraint:
504 """Convert a `ForeignKeySpec` to a
505 `sqlalchemy.schema.ForeignKeyConstraint`.
507 Parameters
508 ----------
509 table : `str`
510 Name of the table this foreign key is being added to.
511 spec : `ForeignKeySpec`
512 Specification for the foreign key to be added.
513 metadata : `sqlalchemy.MetaData`
514 SQLAlchemy representation of the DDL schema this constraint is
515 being added to.
516 **kwds
517 Additional keyword arguments to forward to the
518 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is
519 provided to make it easier for derived classes to delegate to
520 ``super()`` while making only minor changes.
522 Returns
523 -------
524 constraint : `sqlalchemy.schema.ForeignKeyConstraint`
525 SQLAlchemy representation of the constraint.
526 """
527 name = self.shrinkDatabaseEntityName(
528 "_".join(["fkey", table, self._mangleTableName(spec.table)]
529 + list(spec.target) + list(spec.source))
530 )
531 return sqlalchemy.schema.ForeignKeyConstraint(
532 spec.source,
533 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target],
534 name=name,
535 ondelete=spec.onDelete
536 )
538 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData,
539 **kwds) -> sqlalchemy.schema.Table:
540 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`.
542 Parameters
543 ----------
544 spec : `TableSpec`
545 Specification for the foreign key to be added.
546 metadata : `sqlalchemy.MetaData`
547 SQLAlchemy representation of the DDL schema this table is being
548 added to.
549 **kwds
550 Additional keyword arguments to forward to the
551 `sqlalchemy.schema.Table` constructor. This is provided to make it
552 easier for derived classes to delegate to ``super()`` while making
553 only minor changes.
555 Returns
556 -------
557 table : `sqlalchemy.schema.Table`
558 SQLAlchemy representation of the table.
560 Notes
561 -----
562 This method does not handle ``spec.foreignKeys`` at all, in order to
563 avoid circular dependencies. These are added by higher-level logic in
564 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`.
565 """
566 name = self._mangleTableName(name)
567 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields]
568 args.extend(
569 sqlalchemy.schema.UniqueConstraint(
570 *columns,
571 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns)))
572 )
573 for columns in spec.unique if columns not in spec.indexes
574 )
575 args.extend(
576 sqlalchemy.schema.Index(
577 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))),
578 *columns,
579 unique=(columns in spec.unique)
580 )
581 for columns in spec.indexes
582 )
583 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}."
584 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds)
586 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause:
587 """Ensure that a table with the given name and specification exists,
588 creating it if necessary.
590 Parameters
591 ----------
592 name : `str`
593 Name of the table (not including namespace qualifiers).
594 spec : `TableSpec`
595 Specification for the table. This will be used when creating the
596 table, and *may* be used when obtaining an existing table to check
597 for consistency, but no such check is guaranteed.
599 Returns
600 -------
601 table : `sqlalchemy.schema.Table`
602 SQLAlchemy representation of the table.
604 Raises
605 ------
606 ReadOnlyDatabaseError
607 Raised if `isWriteable` returns `False`, and the table does not
608 already exist.
609 DatabaseConflictError
610 Raised if the table exists but ``spec`` is inconsistent with its
611 definition.
613 Notes
614 -----
615 This method may not be called within transactions. It may be called on
616 read-only databases if and only if the table does in fact already
617 exist.
619 Subclasses may override this method, but usually should not need to.
620 """
621 assert not self._connection.in_transaction(), "Table creation interrupts transactions."
622 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
623 table = self.getExistingTable(name, spec)
624 if table is not None:
625 return table
626 if not self.isWriteable():
627 raise ReadOnlyDatabaseError(
628 f"Table {name} does not exist, and cannot be created "
629 f"because database {self} is read-only."
630 )
631 table = self._convertTableSpec(name, spec, self._metadata)
632 for foreignKeySpec in spec.foreignKeys:
633 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
634 table.create(self._connection)
635 return table
637 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]:
638 """Obtain an existing table with the given name and specification.
640 Parameters
641 ----------
642 name : `str`
643 Name of the table (not including namespace qualifiers).
644 spec : `TableSpec`
645 Specification for the table. This will be used when creating the
646 SQLAlchemy representation of the table, and it is used to
647 check that the actual table in the database is consistent.
649 Returns
650 -------
651 table : `sqlalchemy.schema.Table` or `None`
652 SQLAlchemy representation of the table, or `None` if it does not
653 exist.
655 Raises
656 ------
657 DatabaseConflictError
658 Raised if the table exists but ``spec`` is inconsistent with its
659 definition.
661 Notes
662 -----
663 This method can be called within transactions and never modifies the
664 database.
666 Subclasses may override this method, but usually should not need to.
667 """
668 assert self._metadata is not None, "Static tables must be declared before dynamic tables."
669 name = self._mangleTableName(name)
670 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}")
671 if table is not None:
672 if spec.fields.names != set(table.columns.keys()):
673 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new "
674 f"specification has columns {list(spec.fields.names)}, while "
675 f"the previous definition has {list(table.columns.keys())}.")
676 else:
677 inspector = sqlalchemy.engine.reflection.Inspector(self._connection)
678 if name in inspector.get_table_names(schema=self.namespace):
679 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace))
680 table = self._convertTableSpec(name, spec, self._metadata)
681 for foreignKeySpec in spec.foreignKeys:
682 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata))
683 return table
684 return table
686 def sync(self, table: sqlalchemy.schema.Table, *,
687 keys: Dict[str, Any],
688 compared: Optional[Dict[str, Any]] = None,
689 extra: Optional[Dict[str, Any]] = None,
690 returning: Optional[Sequence[str]] = None,
691 ) -> Tuple[Optional[Dict[str, Any], bool]]:
692 """Insert into a table as necessary to ensure database contains
693 values equivalent to the given ones.
695 Parameters
696 ----------
697 table : `sqlalchemy.schema.Table`
698 Table to be queried and possibly inserted into.
699 keys : `dict`
700 Column name-value pairs used to search for an existing row; must
701 be a combination that can be used to select a single row if one
702 exists. If such a row does not exist, these values are used in
703 the insert.
704 compared : `dict`, optional
705 Column name-value pairs that are compared to those in any existing
706 row. If such a row does not exist, these rows are used in the
707 insert.
708 extra : `dict`, optional
709 Column name-value pairs that are ignored if a matching row exists,
710 but used in an insert if one is necessary.
711 returning : `~collections.abc.Sequence` of `str`, optional
712 The names of columns whose values should be returned.
714 Returns
715 -------
716 row : `dict`, optional
717 The value of the fields indicated by ``returning``, or `None` if
718 ``returning`` is `None`.
719 inserted : `bool`
720 If `True`, a new row was inserted.
722 Raises
723 ------
724 DatabaseConflictError
725 Raised if the values in ``compared`` do not match the values in the
726 database.
727 ReadOnlyDatabaseError
728 Raised if `isWriteable` returns `False`, and no matching record
729 already exists.
731 Notes
732 -----
733 This method may not be called within transactions. It may be called on
734 read-only databases if and only if the matching row does in fact
735 already exist.
736 """
738 def check():
739 """Query for a row that matches the ``key`` argument, and compare
740 to what was given by the caller.
742 Returns
743 -------
744 n : `int`
745 Number of matching rows. ``n != 1`` is always an error, but
746 it's a different kind of error depending on where `check` is
747 being called.
748 bad : `list` of `str`, or `None`
749 The subset of the keys of ``compared`` for which the existing
750 values did not match the given one. Once again, ``not bad``
751 is always an error, but a different kind on context. `None`
752 if ``n != 1``
753 result : `list` or `None`
754 Results in the database that correspond to the columns given
755 in ``returning``, or `None` if ``returning is None``.
756 """
757 toSelect = set()
758 if compared is not None:
759 toSelect.update(compared.keys())
760 if returning is not None:
761 toSelect.update(returning)
762 if not toSelect:
763 # Need to select some column, even if we just want to see
764 # how many rows we get back.
765 toSelect.add(next(iter(keys.keys())))
766 selectSql = sqlalchemy.sql.select(
767 [table.columns[k].label(k) for k in toSelect]
768 ).select_from(table).where(
769 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])
770 )
771 fetched = list(self._connection.execute(selectSql).fetchall())
772 if len(fetched) != 1:
773 return len(fetched), None, None
774 existing = fetched[0]
775 if compared is not None:
777 def safeNotEqual(a, b):
778 if isinstance(a, astropy.time.Time):
779 return not time_utils.times_equal(a, b)
780 return a != b
782 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}"
783 for k, v in compared.items()
784 if safeNotEqual(existing[k], v)]
785 else:
786 inconsistencies = []
787 if returning is not None:
788 toReturn = [existing[k] for k in returning]
789 else:
790 toReturn = None
791 return 1, inconsistencies, toReturn
793 if self.isWriteable():
794 # Database is writeable. Try an insert first, but allow it to fail
795 # (in only specific ways).
796 row = keys.copy()
797 if compared is not None:
798 row.update(compared)
799 if extra is not None:
800 row.update(extra)
801 insertSql = table.insert().values(row)
802 try:
803 with self.transaction(interrupting=True):
804 self._connection.execute(insertSql)
805 # Need to perform check() for this branch inside the
806 # transaction, so we roll back an insert that didn't do
807 # what we expected. That limits the extent to which we
808 # can reduce duplication between this block and the other
809 # ones that perform similar logic.
810 n, bad, result = check()
811 if n < 1:
812 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.")
813 elif n > 2:
814 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
815 f"unique constraint for table {table.name}.")
816 elif bad:
817 raise RuntimeError(
818 f"Conflict ({bad}) in sync after successful insert; this is "
819 f"possible if the same table is being updated by a concurrent "
820 f"process that isn't using sync, but it may also be a bug in "
821 f"daf_butler."
822 )
823 # No exceptions, so it looks like we inserted the requested row
824 # successfully.
825 inserted = True
826 except sqlalchemy.exc.IntegrityError as err:
827 # Most likely cause is that an equivalent row already exists,
828 # but it could also be some other constraint. Query for the
829 # row we think we matched to resolve that question.
830 n, bad, result = check()
831 if n < 1:
832 # There was no matched row; insertion failed for some
833 # completely different reason. Just re-raise the original
834 # IntegrityError.
835 raise
836 elif n > 2:
837 # There were multiple matched rows, which means we
838 # conflicted *and* the arguments were bad to begin with.
839 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a "
840 f"unique constraint for table {table.name}.") from err
841 elif bad:
842 # No logic bug, but data conflicted on the keys given.
843 raise DatabaseConflictError(f"Conflict in sync for table "
844 f"{table.name} on column(s) {bad}.") from err
845 # The desired row is already present and consistent with what
846 # we tried to insert.
847 inserted = False
848 else:
849 assert not self._connection.in_transaction(), (
850 "Calling sync within a transaction block is an error even "
851 "on a read-only database."
852 )
853 # Database is not writeable; just see if the row exists.
854 n, bad, result = check()
855 if n < 1:
856 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.")
857 elif n > 1:
858 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.")
859 elif bad:
860 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.")
861 inserted = False
862 return {k: v for k, v in zip(returning, result)} if returning is not None else None, inserted
864 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False,
865 ) -> Optional[List[int]]:
866 """Insert one or more rows into a table, optionally returning
867 autoincrement primary key values.
869 Parameters
870 ----------
871 table : `sqlalchemy.schema.Table`
872 Table rows should be inserted into.
873 returnIds: `bool`
874 If `True` (`False` is default), return the values of the table's
875 autoincrement primary key field (which much exist).
876 *rows
877 Positional arguments are the rows to be inserted, as dictionaries
878 mapping column name to value. The keys in all dictionaries must
879 be the same.
881 Returns
882 -------
883 ids : `None`, or `list` of `int`
884 If ``returnIds`` is `True`, a `list` containing the inserted
885 values for the table's autoincrement primary key.
887 Raises
888 ------
889 ReadOnlyDatabaseError
890 Raised if `isWriteable` returns `False` when this method is called.
892 Notes
893 -----
894 The default implementation uses bulk insert syntax when ``returnIds``
895 is `False`, and a loop over single-row insert operations when it is
896 `True`.
898 Derived classes should reimplement when they can provide a more
899 efficient implementation (especially for the latter case).
901 May be used inside transaction contexts, so implementations may not
902 perform operations that interrupt transactions.
903 """
904 if not self.isWriteable():
905 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.")
906 if not returnIds:
907 self._connection.execute(table.insert(), *rows)
908 else:
909 sql = table.insert()
910 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows]
912 @abstractmethod
913 def replace(self, table: sqlalchemy.schema.Table, *rows: dict):
914 """Insert one or more rows into a table, replacing any existing rows
915 for which insertion of a new row would violate the primary key
916 constraint.
918 Parameters
919 ----------
920 table : `sqlalchemy.schema.Table`
921 Table rows should be inserted into.
922 *rows
923 Positional arguments are the rows to be inserted, as dictionaries
924 mapping column name to value. The keys in all dictionaries must
925 be the same.
927 Raises
928 ------
929 ReadOnlyDatabaseError
930 Raised if `isWriteable` returns `False` when this method is called.
932 Notes
933 -----
934 May be used inside transaction contexts, so implementations may not
935 perform operations that interrupt transactions.
937 Implementations should raise a `sqlalchemy.exc.IntegrityError`
938 exception when a constraint other than the primary key would be
939 violated.
941 Implementations are not required to support `replace` on tables
942 with autoincrement keys.
943 """
944 raise NotImplementedError()
946 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int:
947 """Delete one or more rows from a table.
949 Parameters
950 ----------
951 table : `sqlalchemy.schema.Table`
952 Table that rows should be deleted from.
953 columns: `~collections.abc.Iterable` of `str`
954 The names of columns that will be used to constrain the rows to
955 be deleted; these will be combined via ``AND`` to form the
956 ``WHERE`` clause of the delete query.
957 *rows
958 Positional arguments are the keys of rows to be deleted, as
959 dictionaries mapping column name to value. The keys in all
960 dictionaries must exactly the names in ``columns``.
962 Returns
963 -------
964 count : `int`
965 Number of rows deleted.
967 Raises
968 ------
969 ReadOnlyDatabaseError
970 Raised if `isWriteable` returns `False` when this method is called.
972 Notes
973 -----
974 May be used inside transaction contexts, so implementations may not
975 perform operations that interrupt transactions.
977 The default implementation should be sufficient for most derived
978 classes.
979 """
980 if not self.isWriteable():
981 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.")
982 sql = table.delete()
983 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns]
984 if whereTerms:
985 sql = sql.where(sqlalchemy.sql.and_(*whereTerms))
986 return self._connection.execute(sql, *rows).rowcount
988 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int:
989 """Update one or more rows in a table.
991 Parameters
992 ----------
993 table : `sqlalchemy.schema.Table`
994 Table containing the rows to be updated.
995 where : `dict` [`str`, `str`]
996 A mapping from the names of columns that will be used to search for
997 existing rows to the keys that will hold these values in the
998 ``rows`` dictionaries. Note that these may not be the same due to
999 SQLAlchemy limitations.
1000 *rows
1001 Positional arguments are the rows to be updated. The keys in all
1002 dictionaries must be the same, and may correspond to either a
1003 value in the ``where`` dictionary or the name of a column to be
1004 updated.
1006 Returns
1007 -------
1008 count : `int`
1009 Number of rows matched (regardless of whether the update actually
1010 modified them).
1012 Raises
1013 ------
1014 ReadOnlyDatabaseError
1015 Raised if `isWriteable` returns `False` when this method is called.
1017 Notes
1018 -----
1019 May be used inside transaction contexts, so implementations may not
1020 perform operations that interrupt transactions.
1022 The default implementation should be sufficient for most derived
1023 classes.
1024 """
1025 if not self.isWriteable():
1026 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.")
1027 sql = table.update().where(
1028 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()])
1029 )
1030 return self._connection.execute(sql, *rows).rowcount
1032 def query(self, sql: sqlalchemy.sql.FromClause, *args, **kwds) -> sqlalchemy.engine.ResultProxy:
1033 """Run a SELECT query against the database.
1035 Parameters
1036 ----------
1037 sql : `sqlalchemy.sql.FromClause`
1038 A SQLAlchemy representation of a ``SELECT`` query.
1039 *args
1040 Additional positional arguments are forwarded to
1041 `sqlalchemy.engine.Connection.execute`.
1042 **kwds
1043 Additional keyword arguments are forwarded to
1044 `sqlalchemy.engine.Connection.execute`.
1046 Returns
1047 -------
1048 result : `sqlalchemy.engine.ResultProxy`
1049 Query results.
1051 Notes
1052 -----
1053 The default implementation should be sufficient for most derived
1054 classes.
1055 """
1056 # TODO: should we guard against non-SELECT queries here?
1057 return self._connection.execute(sql, *args, **kwds)
1059 origin: int
1060 """An integer ID that should be used as the default for any datasets,
1061 quanta, or other entities that use a (autoincrement, origin) compound
1062 primary key (`int`).
1063 """
1065 namespace: Optional[str]
1066 """The schema or namespace this database instance is associated with
1067 (`str` or `None`).
1068 """