Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "SchemaAlreadyDefinedError", 

28 "StaticTablesContext", 

29] 

30 

31from abc import ABC, abstractmethod 

32from contextlib import contextmanager 

33from typing import ( 

34 Any, 

35 Callable, 

36 Dict, 

37 Iterable, 

38 Iterator, 

39 List, 

40 Optional, 

41 Sequence, 

42 Set, 

43 Tuple, 

44 Type, 

45 Union, 

46) 

47import uuid 

48import warnings 

49 

50import astropy.time 

51import sqlalchemy 

52 

53from ...core import DatabaseTimespanRepresentation, ddl, time_utils 

54from .._exceptions import ConflictingDefinitionError 

55 

56_IN_SAVEPOINT_TRANSACTION = "IN_SAVEPOINT_TRANSACTION" 

57 

58 

59def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None: 

60 """Test that the definition of a table in a `ddl.TableSpec` and from 

61 database introspection are consistent. 

62 

63 Parameters 

64 ---------- 

65 name : `str` 

66 Name of the table (only used in error messages). 

67 spec : `ddl.TableSpec` 

68 Specification of the table. 

69 inspection : `dict` 

70 Dictionary returned by 

71 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

72 

73 Raises 

74 ------ 

75 DatabaseConflictError 

76 Raised if the definitions are inconsistent. 

77 """ 

78 columnNames = [c["name"] for c in inspection] 

79 if spec.fields.names != set(columnNames): 

80 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

81 f"specification has columns {list(spec.fields.names)}, while the " 

82 f"table in the database has {columnNames}.") 

83 

84 

85class ReadOnlyDatabaseError(RuntimeError): 

86 """Exception raised when a write operation is called on a read-only 

87 `Database`. 

88 """ 

89 

90 

91class DatabaseConflictError(ConflictingDefinitionError): 

92 """Exception raised when database content (row values or schema entities) 

93 are inconsistent with what this client expects. 

94 """ 

95 

96 

97class SchemaAlreadyDefinedError(RuntimeError): 

98 """Exception raised when trying to initialize database schema when some 

99 tables already exist. 

100 """ 

101 

102 

103class StaticTablesContext: 

104 """Helper class used to declare the static schema for a registry layer 

105 in a database. 

106 

107 An instance of this class is returned by `Database.declareStaticTables`, 

108 which should be the only way it should be constructed. 

109 """ 

110 

111 def __init__(self, db: Database): 

112 self._db = db 

113 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = [] 

114 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

115 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

116 self._initializers: List[Callable[[Database], None]] = [] 

117 

118 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

119 """Add a new table to the schema, returning its sqlalchemy 

120 representation. 

121 

122 The new table may not actually be created until the end of the 

123 context created by `Database.declareStaticTables`, allowing tables 

124 to be declared in any order even in the presence of foreign key 

125 relationships. 

126 """ 

127 name = self._db._mangleTableName(name) 

128 if name in self._tableNames: 

129 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

130 schema=self._db.namespace)) 

131 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

132 for foreignKeySpec in spec.foreignKeys: 

133 self._foreignKeys.append( 

134 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

135 ) 

136 return table 

137 

138 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

139 """Add a named tuple of tables to the schema, returning their 

140 SQLAlchemy representations in a named tuple of the same type. 

141 

142 The new tables may not actually be created until the end of the 

143 context created by `Database.declareStaticTables`, allowing tables 

144 to be declared in any order even in the presence of foreign key 

145 relationships. 

146 

147 Notes 

148 ----- 

149 ``specs`` *must* be an instance of a type created by 

150 `collections.namedtuple`, not just regular tuple, and the returned 

151 object is guaranteed to be the same. Because `~collections.namedtuple` 

152 is just a factory for `type` objects, not an actual type itself, 

153 we cannot represent this with type annotations. 

154 """ 

155 return specs._make(self.addTable(name, spec) # type: ignore 

156 for name, spec in zip(specs._fields, specs)) # type: ignore 

157 

158 def addInitializer(self, initializer: Callable[[Database], None]) -> None: 

159 """Add a method that does one-time initialization of a database. 

160 

161 Initialization can mean anything that changes state of a database 

162 and needs to be done exactly once after database schema was created. 

163 An example for that could be population of schema attributes. 

164 

165 Parameters 

166 ---------- 

167 initializer : callable 

168 Method of a single argument which is a `Database` instance. 

169 """ 

170 self._initializers.append(initializer) 

171 

172 

173class Database(ABC): 

174 """An abstract interface that represents a particular database engine's 

175 representation of a single schema/namespace/database. 

176 

177 Parameters 

178 ---------- 

179 origin : `int` 

180 An integer ID that should be used as the default for any datasets, 

181 quanta, or other entities that use a (autoincrement, origin) compound 

182 primary key. 

183 connection : `sqlalchemy.engine.Connection` 

184 The SQLAlchemy connection this `Database` wraps. 

185 namespace : `str`, optional 

186 Name of the schema or namespace this instance is associated with. 

187 This is passed as the ``schema`` argument when constructing a 

188 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

189 avoid confusion between "schema means namespace" and "schema means 

190 table definitions". 

191 

192 Notes 

193 ----- 

194 `Database` requires all write operations to go through its special named 

195 methods. Our write patterns are sufficiently simple that we don't really 

196 need the full flexibility of SQL insert/update/delete syntax, and we need 

197 non-standard (but common) functionality in these operations sufficiently 

198 often that it seems worthwhile to provide our own generic API. 

199 

200 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

201 their SQLAlchemy representation) to be run, as we expect these to require 

202 significantly more sophistication while still being limited to standard 

203 SQL. 

204 

205 `Database` itself has several underscore-prefixed attributes: 

206 

207 - ``_connection``: SQLAlchemy object representing the connection. 

208 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

209 the tables and other schema entities. 

210 

211 These are considered protected (derived classes may access them, but other 

212 code should not), and read-only, aside from executing SQL via 

213 ``_connection``. 

214 """ 

215 

216 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection, 

217 namespace: Optional[str] = None): 

218 self.origin = origin 

219 self.namespace = namespace 

220 self._connection = connection 

221 self._metadata: Optional[sqlalchemy.schema.MetaData] = None 

222 self._tempTables: Set[str] = set() 

223 

224 def __repr__(self) -> str: 

225 # Rather than try to reproduce all the parameters used to create 

226 # the object, instead report the more useful information of the 

227 # connection URL. 

228 uri = str(self._connection.engine.url) 

229 if self.namespace: 

230 uri += f"#{self.namespace}" 

231 return f'{type(self).__name__}("{uri}")' 

232 

233 @classmethod 

234 def makeDefaultUri(cls, root: str) -> Optional[str]: 

235 """Create a default connection URI appropriate for the given root 

236 directory, or `None` if there can be no such default. 

237 """ 

238 return None 

239 

240 @classmethod 

241 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

242 writeable: bool = True) -> Database: 

243 """Construct a database from a SQLAlchemy URI. 

244 

245 Parameters 

246 ---------- 

247 uri : `str` 

248 A SQLAlchemy URI connection string. 

249 origin : `int` 

250 An integer ID that should be used as the default for any datasets, 

251 quanta, or other entities that use a (autoincrement, origin) 

252 compound primary key. 

253 namespace : `str`, optional 

254 A database namespace (i.e. schema) the new instance should be 

255 associated with. If `None` (default), the namespace (if any) is 

256 inferred from the URI. 

257 writeable : `bool`, optional 

258 If `True`, allow write operations on the database, including 

259 ``CREATE TABLE``. 

260 

261 Returns 

262 ------- 

263 db : `Database` 

264 A new `Database` instance. 

265 """ 

266 return cls.fromConnection(cls.connect(uri, writeable=writeable), 

267 origin=origin, 

268 namespace=namespace, 

269 writeable=writeable) 

270 

271 @classmethod 

272 @abstractmethod 

273 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection: 

274 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI. 

275 

276 Parameters 

277 ---------- 

278 uri : `str` 

279 A SQLAlchemy URI connection string. 

280 origin : `int` 

281 An integer ID that should be used as the default for any datasets, 

282 quanta, or other entities that use a (autoincrement, origin) 

283 compound primary key. 

284 writeable : `bool`, optional 

285 If `True`, allow write operations on the database, including 

286 ``CREATE TABLE``. 

287 

288 Returns 

289 ------- 

290 connection : `sqlalchemy.engine.Connection` 

291 A database connection. 

292 

293 Notes 

294 ----- 

295 Subclasses that support other ways to connect to a database are 

296 encouraged to add optional arguments to their implementation of this 

297 method, as long as they maintain compatibility with the base class 

298 call signature. 

299 """ 

300 raise NotImplementedError() 

301 

302 @classmethod 

303 @abstractmethod 

304 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int, 

305 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

306 """Create a new `Database` from an existing 

307 `sqlalchemy.engine.Connection`. 

308 

309 Parameters 

310 ---------- 

311 connection : `sqllachemy.engine.Connection` 

312 The connection for the the database. May be shared between 

313 `Database` instances. 

314 origin : `int` 

315 An integer ID that should be used as the default for any datasets, 

316 quanta, or other entities that use a (autoincrement, origin) 

317 compound primary key. 

318 namespace : `str`, optional 

319 A different database namespace (i.e. schema) the new instance 

320 should be associated with. If `None` (default), the namespace 

321 (if any) is inferred from the connection. 

322 writeable : `bool`, optional 

323 If `True`, allow write operations on the database, including 

324 ``CREATE TABLE``. 

325 

326 Returns 

327 ------- 

328 db : `Database` 

329 A new `Database` instance. 

330 

331 Notes 

332 ----- 

333 This method allows different `Database` instances to share the same 

334 connection, which is desirable when they represent different namespaces 

335 can be queried together. This also ties their transaction state, 

336 however; starting a transaction in any database automatically starts 

337 on in all other databases. 

338 """ 

339 raise NotImplementedError() 

340 

341 @contextmanager 

342 def transaction(self, *, interrupting: bool = False, savepoint: bool = False, 

343 lock: Iterable[sqlalchemy.schema.Table] = ()) -> Iterator: 

344 """Return a context manager that represents a transaction. 

345 

346 Parameters 

347 ---------- 

348 interrupting : `bool`, optional 

349 If `True` (`False` is default), this transaction block may not be 

350 nested without an outer one, and attempting to do so is a logic 

351 (i.e. assertion) error. 

352 savepoint : `bool`, optional 

353 If `True` (`False` is default), create a `SAVEPOINT`, allowing 

354 exceptions raised by the database (e.g. due to constraint 

355 violations) during this transaction's context to be caught outside 

356 it without also rolling back all operations in an outer transaction 

357 block. If `False`, transactions may still be nested, but a 

358 rollback may be generated at any level and affects all levels, and 

359 commits are deferred until the outermost block completes. If any 

360 outer transaction block was created with ``savepoint=True``, all 

361 inner blocks will be as well (regardless of the actual value 

362 passed). This has no effect if this is the outermost transaction. 

363 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

364 A list of tables to lock for the duration of this transaction. 

365 These locks are guaranteed to prevent concurrent writes, but only 

366 prevent concurrent reads if the database engine requires that in 

367 order to block concurrent writes. 

368 

369 Notes 

370 ----- 

371 All transactions on a connection managed by one or more `Database` 

372 instances _must_ go through this method, or transaction state will not 

373 be correctly managed. 

374 """ 

375 assert not (interrupting and self._connection.in_transaction()), ( 

376 "Logic error in transaction nesting: an operation that would " 

377 "interrupt the active transaction context has been requested." 

378 ) 

379 # We remember whether we are already in a SAVEPOINT transaction via the 

380 # connection object's 'info' dict, which is explicitly for user 

381 # information like this. This is safer than a regular `Database` 

382 # instance attribute, because it guards against multiple `Database` 

383 # instances sharing the same connection. The need to use our own flag 

384 # here to track whether we're in a nested transaction should go away in 

385 # SQLAlchemy 1.4, which seems to have a 

386 # `Connection.in_nested_transaction()` method. 

387 savepoint = savepoint or self._connection.info.get(_IN_SAVEPOINT_TRANSACTION, False) 

388 self._connection.info[_IN_SAVEPOINT_TRANSACTION] = savepoint 

389 if self._connection.in_transaction() and savepoint: 

390 trans = self._connection.begin_nested() 

391 else: 

392 # Use a regular (non-savepoint) transaction always for the 

393 # outermost context, as well as when a savepoint was not requested. 

394 trans = self._connection.begin() 

395 self._lockTables(lock) 

396 try: 

397 yield 

398 trans.commit() 

399 except BaseException: 

400 trans.rollback() 

401 raise 

402 finally: 

403 if not self._connection.in_transaction(): 

404 self._connection.info.pop(_IN_SAVEPOINT_TRANSACTION, None) 

405 

406 @abstractmethod 

407 def _lockTables(self, tables: Iterable[sqlalchemy.schema.Table] = ()) -> None: 

408 """Acquire locks on the given tables. 

409 

410 This is an implementation hook for subclasses, called by `transaction`. 

411 It should not be called directly by other code. 

412 

413 Parameters 

414 ---------- 

415 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

416 A list of tables to lock for the duration of this transaction. 

417 These locks are guaranteed to prevent concurrent writes, but only 

418 prevent concurrent reads if the database engine requires that in 

419 order to block concurrent writes. 

420 """ 

421 raise NotImplementedError() 

422 

423 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool: 

424 """Check whether a table is writeable, either because the database 

425 connection is read-write or the table is a temporary table. 

426 

427 Parameters 

428 ---------- 

429 table : `sqlalchemy.schema.Table` 

430 SQLAlchemy table object to check. 

431 

432 Returns 

433 ------- 

434 writeable : `bool` 

435 Whether this table is writeable. 

436 """ 

437 return self.isWriteable() or table.key in self._tempTables 

438 

439 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None: 

440 """Raise if the given table is not writeable, either because the 

441 database connection is read-write or the table is a temporary table. 

442 

443 Parameters 

444 ---------- 

445 table : `sqlalchemy.schema.Table` 

446 SQLAlchemy table object to check. 

447 msg : `str`, optional 

448 If provided, raise `ReadOnlyDatabaseError` instead of returning 

449 `False`, with this message. 

450 """ 

451 if not self.isTableWriteable(table): 

452 raise ReadOnlyDatabaseError(msg) 

453 

454 @contextmanager 

455 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]: 

456 """Return a context manager in which the database's static DDL schema 

457 can be declared. 

458 

459 Parameters 

460 ---------- 

461 create : `bool` 

462 If `True`, attempt to create all tables at the end of the context. 

463 If `False`, they will be assumed to already exist. 

464 

465 Returns 

466 ------- 

467 schema : `StaticTablesContext` 

468 A helper object that is used to add new tables. 

469 

470 Raises 

471 ------ 

472 ReadOnlyDatabaseError 

473 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

474 and one or more declared tables do not already exist. 

475 

476 Examples 

477 -------- 

478 Given a `Database` instance ``db``:: 

479 

480 with db.declareStaticTables(create=True) as schema: 

481 schema.addTable("table1", TableSpec(...)) 

482 schema.addTable("table2", TableSpec(...)) 

483 

484 Notes 

485 ----- 

486 A database's static DDL schema must be declared before any dynamic 

487 tables are managed via calls to `ensureTableExists` or 

488 `getExistingTable`. The order in which static schema tables are added 

489 inside the context block is unimportant; they will automatically be 

490 sorted and added in an order consistent with their foreign key 

491 relationships. 

492 """ 

493 if create and not self.isWriteable(): 

494 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

495 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

496 try: 

497 context = StaticTablesContext(self) 

498 if create and context._tableNames: 

499 # Looks like database is already initalized, to avoid danger 

500 # of modifying/destroying valid schema we refuse to do 

501 # anything in this case 

502 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.") 

503 yield context 

504 for table, foreignKey in context._foreignKeys: 

505 table.append_constraint(foreignKey) 

506 if create: 

507 if self.namespace is not None: 

508 if self.namespace not in context._inspector.get_schema_names(): 

509 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

510 # In our tables we have columns that make use of sqlalchemy 

511 # Sequence objects. There is currently a bug in sqlalchemy that 

512 # causes a deprecation warning to be thrown on a property of 

513 # the Sequence object when the repr for the sequence is 

514 # created. Here a filter is used to catch these deprecation 

515 # warnings when tables are created. 

516 with warnings.catch_warnings(): 

517 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

518 self._metadata.create_all(self._connection) 

519 # call all initializer methods sequentially 

520 for init in context._initializers: 

521 init(self) 

522 except BaseException: 

523 self._metadata = None 

524 raise 

525 

526 @abstractmethod 

527 def isWriteable(self) -> bool: 

528 """Return `True` if this database can be modified by this client. 

529 """ 

530 raise NotImplementedError() 

531 

532 @abstractmethod 

533 def __str__(self) -> str: 

534 """Return a human-readable identifier for this `Database`, including 

535 any namespace or schema that identifies its names within a `Registry`. 

536 """ 

537 raise NotImplementedError() 

538 

539 @property 

540 def dialect(self) -> sqlalchemy.engine.Dialect: 

541 """The SQLAlchemy dialect for this database engine 

542 (`sqlalchemy.engine.Dialect`). 

543 """ 

544 return self._connection.dialect 

545 

546 def shrinkDatabaseEntityName(self, original: str) -> str: 

547 """Return a version of the given name that fits within this database 

548 engine's length limits for table, constraint, indexes, and sequence 

549 names. 

550 

551 Implementations should not assume that simple truncation is safe, 

552 because multiple long names often begin with the same prefix. 

553 

554 The default implementation simply returns the given name. 

555 

556 Parameters 

557 ---------- 

558 original : `str` 

559 The original name. 

560 

561 Returns 

562 ------- 

563 shrunk : `str` 

564 The new, possibly shortened name. 

565 """ 

566 return original 

567 

568 def expandDatabaseEntityName(self, shrunk: str) -> str: 

569 """Retrieve the original name for a database entity that was too long 

570 to fit within the database engine's limits. 

571 

572 Parameters 

573 ---------- 

574 original : `str` 

575 The original name. 

576 

577 Returns 

578 ------- 

579 shrunk : `str` 

580 The new, possibly shortened name. 

581 """ 

582 return shrunk 

583 

584 def _mangleTableName(self, name: str) -> str: 

585 """Map a logical, user-visible table name to the true table name used 

586 in the database. 

587 

588 The default implementation returns the given name unchanged. 

589 

590 Parameters 

591 ---------- 

592 name : `str` 

593 Input table name. Should not include a namespace (i.e. schema) 

594 prefix. 

595 

596 Returns 

597 ------- 

598 mangled : `str` 

599 Mangled version of the table name (still with no namespace prefix). 

600 

601 Notes 

602 ----- 

603 Reimplementations of this method must be idempotent - mangling an 

604 already-mangled name must have no effect. 

605 """ 

606 return name 

607 

608 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]: 

609 """Create constraints based on this spec. 

610 

611 Parameters 

612 ---------- 

613 table : `str` 

614 Name of the table this column is being added to. 

615 spec : `FieldSpec` 

616 Specification for the field to be added. 

617 

618 Returns 

619 ------- 

620 constraint : `list` of `sqlalchemy.CheckConstraint` 

621 Constraint added for this column. 

622 """ 

623 # By default we return no additional constraints 

624 return [] 

625 

626 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

627 **kwds: Any) -> sqlalchemy.schema.Column: 

628 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

629 

630 Parameters 

631 ---------- 

632 table : `str` 

633 Name of the table this column is being added to. 

634 spec : `FieldSpec` 

635 Specification for the field to be added. 

636 metadata : `sqlalchemy.MetaData` 

637 SQLAlchemy representation of the DDL schema this field's table is 

638 being added to. 

639 **kwds 

640 Additional keyword arguments to forward to the 

641 `sqlalchemy.schema.Column` constructor. This is provided to make 

642 it easier for derived classes to delegate to ``super()`` while 

643 making only minor changes. 

644 

645 Returns 

646 ------- 

647 column : `sqlalchemy.schema.Column` 

648 SQLAlchemy representation of the field. 

649 """ 

650 args = [spec.name, spec.getSizedColumnType()] 

651 if spec.autoincrement: 

652 # Generate a sequence to use for auto incrementing for databases 

653 # that do not support it natively. This will be ignored by 

654 # sqlalchemy for databases that do support it. 

655 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

656 metadata=metadata)) 

657 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

658 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

659 comment=spec.doc, server_default=spec.default, **kwds) 

660 

661 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

662 **kwds: Any) -> sqlalchemy.schema.ForeignKeyConstraint: 

663 """Convert a `ForeignKeySpec` to a 

664 `sqlalchemy.schema.ForeignKeyConstraint`. 

665 

666 Parameters 

667 ---------- 

668 table : `str` 

669 Name of the table this foreign key is being added to. 

670 spec : `ForeignKeySpec` 

671 Specification for the foreign key to be added. 

672 metadata : `sqlalchemy.MetaData` 

673 SQLAlchemy representation of the DDL schema this constraint is 

674 being added to. 

675 **kwds 

676 Additional keyword arguments to forward to the 

677 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

678 provided to make it easier for derived classes to delegate to 

679 ``super()`` while making only minor changes. 

680 

681 Returns 

682 ------- 

683 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

684 SQLAlchemy representation of the constraint. 

685 """ 

686 name = self.shrinkDatabaseEntityName( 

687 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

688 + list(spec.target) + list(spec.source)) 

689 ) 

690 return sqlalchemy.schema.ForeignKeyConstraint( 

691 spec.source, 

692 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

693 name=name, 

694 ondelete=spec.onDelete 

695 ) 

696 

697 def _convertExclusionConstraintSpec(self, table: str, 

698 spec: Tuple[Union[str, Type[DatabaseTimespanRepresentation]], ...], 

699 metadata: sqlalchemy.MetaData) -> sqlalchemy.schema.Constraint: 

700 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy 

701 constraint representation. 

702 

703 Parameters 

704 ---------- 

705 table : `str` 

706 Name of the table this constraint is being added to. 

707 spec : `tuple` [ `str` or `type` ] 

708 A tuple of `str` column names and the `type` object returned by 

709 `getTimespanRepresentation` (which must appear exactly once), 

710 indicating the order of the columns in the index used to back the 

711 constraint. 

712 metadata : `sqlalchemy.MetaData` 

713 SQLAlchemy representation of the DDL schema this constraint is 

714 being added to. 

715 

716 Returns 

717 ------- 

718 constraint : `sqlalchemy.schema.Constraint` 

719 SQLAlchemy representation of the constraint. 

720 

721 Raises 

722 ------ 

723 NotImplementedError 

724 Raised if this database does not support exclusion constraints. 

725 """ 

726 raise NotImplementedError(f"Database {self} does not support exclusion constraints.") 

727 

728 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

729 **kwds: Any) -> sqlalchemy.schema.Table: 

730 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

731 

732 Parameters 

733 ---------- 

734 spec : `TableSpec` 

735 Specification for the foreign key to be added. 

736 metadata : `sqlalchemy.MetaData` 

737 SQLAlchemy representation of the DDL schema this table is being 

738 added to. 

739 **kwds 

740 Additional keyword arguments to forward to the 

741 `sqlalchemy.schema.Table` constructor. This is provided to make it 

742 easier for derived classes to delegate to ``super()`` while making 

743 only minor changes. 

744 

745 Returns 

746 ------- 

747 table : `sqlalchemy.schema.Table` 

748 SQLAlchemy representation of the table. 

749 

750 Notes 

751 ----- 

752 This method does not handle ``spec.foreignKeys`` at all, in order to 

753 avoid circular dependencies. These are added by higher-level logic in 

754 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

755 """ 

756 name = self._mangleTableName(name) 

757 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

758 

759 # Add any column constraints 

760 for fieldSpec in spec.fields: 

761 args.extend(self._makeColumnConstraints(name, fieldSpec)) 

762 

763 # Track indexes added for primary key and unique constraints, to make 

764 # sure we don't add duplicate explicit or foreign key indexes for 

765 # those. 

766 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)} 

767 args.extend( 

768 sqlalchemy.schema.UniqueConstraint( 

769 *columns, 

770 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

771 ) 

772 for columns in spec.unique 

773 ) 

774 allIndexes.update(spec.unique) 

775 args.extend( 

776 sqlalchemy.schema.Index( 

777 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

778 *columns, 

779 unique=(columns in spec.unique) 

780 ) 

781 for columns in spec.indexes if columns not in allIndexes 

782 ) 

783 allIndexes.update(spec.indexes) 

784 args.extend( 

785 sqlalchemy.schema.Index( 

786 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)), 

787 *fk.source, 

788 ) 

789 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes 

790 ) 

791 

792 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion) 

793 

794 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

795 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

796 

797 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

798 """Ensure that a table with the given name and specification exists, 

799 creating it if necessary. 

800 

801 Parameters 

802 ---------- 

803 name : `str` 

804 Name of the table (not including namespace qualifiers). 

805 spec : `TableSpec` 

806 Specification for the table. This will be used when creating the 

807 table, and *may* be used when obtaining an existing table to check 

808 for consistency, but no such check is guaranteed. 

809 

810 Returns 

811 ------- 

812 table : `sqlalchemy.schema.Table` 

813 SQLAlchemy representation of the table. 

814 

815 Raises 

816 ------ 

817 ReadOnlyDatabaseError 

818 Raised if `isWriteable` returns `False`, and the table does not 

819 already exist. 

820 DatabaseConflictError 

821 Raised if the table exists but ``spec`` is inconsistent with its 

822 definition. 

823 

824 Notes 

825 ----- 

826 This method may not be called within transactions. It may be called on 

827 read-only databases if and only if the table does in fact already 

828 exist. 

829 

830 Subclasses may override this method, but usually should not need to. 

831 """ 

832 assert not self._connection.in_transaction(), "Table creation interrupts transactions." 

833 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

834 table = self.getExistingTable(name, spec) 

835 if table is not None: 

836 return table 

837 if not self.isWriteable(): 

838 raise ReadOnlyDatabaseError( 

839 f"Table {name} does not exist, and cannot be created " 

840 f"because database {self} is read-only." 

841 ) 

842 table = self._convertTableSpec(name, spec, self._metadata) 

843 for foreignKeySpec in spec.foreignKeys: 

844 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

845 table.create(self._connection) 

846 return table 

847 

848 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

849 """Obtain an existing table with the given name and specification. 

850 

851 Parameters 

852 ---------- 

853 name : `str` 

854 Name of the table (not including namespace qualifiers). 

855 spec : `TableSpec` 

856 Specification for the table. This will be used when creating the 

857 SQLAlchemy representation of the table, and it is used to 

858 check that the actual table in the database is consistent. 

859 

860 Returns 

861 ------- 

862 table : `sqlalchemy.schema.Table` or `None` 

863 SQLAlchemy representation of the table, or `None` if it does not 

864 exist. 

865 

866 Raises 

867 ------ 

868 DatabaseConflictError 

869 Raised if the table exists but ``spec`` is inconsistent with its 

870 definition. 

871 

872 Notes 

873 ----- 

874 This method can be called within transactions and never modifies the 

875 database. 

876 

877 Subclasses may override this method, but usually should not need to. 

878 """ 

879 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

880 name = self._mangleTableName(name) 

881 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

882 if table is not None: 

883 if spec.fields.names != set(table.columns.keys()): 

884 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

885 f"specification has columns {list(spec.fields.names)}, while " 

886 f"the previous definition has {list(table.columns.keys())}.") 

887 else: 

888 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

889 if name in inspector.get_table_names(schema=self.namespace): 

890 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

891 table = self._convertTableSpec(name, spec, self._metadata) 

892 for foreignKeySpec in spec.foreignKeys: 

893 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

894 return table 

895 return table 

896 

897 def makeTemporaryTable(self, spec: ddl.TableSpec, name: Optional[str] = None) -> sqlalchemy.schema.Table: 

898 """Create a temporary table. 

899 

900 Parameters 

901 ---------- 

902 spec : `TableSpec` 

903 Specification for the table. 

904 name : `str`, optional 

905 A unique (within this session/connetion) name for the table. 

906 Subclasses may override to modify the actual name used. If not 

907 provided, a unique name will be generated. 

908 

909 Returns 

910 ------- 

911 table : `sqlalchemy.schema.Table` 

912 SQLAlchemy representation of the table. 

913 

914 Notes 

915 ----- 

916 Temporary tables may be created, dropped, and written to even in 

917 read-only databases - at least according to the Python-level 

918 protections in the `Database` classes. Server permissions may say 

919 otherwise, but in that case they probably need to be modified to 

920 support the full range of expected read-only butler behavior. 

921 

922 Temporary table rows are guaranteed to be dropped when a connection is 

923 closed. `Database` implementations are permitted to allow the table to 

924 remain as long as this is transparent to the user (i.e. "creating" the 

925 temporary table in a new session should not be an error, even if it 

926 does nothing). 

927 

928 It may not be possible to use temporary tables within transactions with 

929 some database engines (or configurations thereof). 

930 """ 

931 if name is None: 

932 name = f"tmp_{uuid.uuid4().hex}" 

933 table = self._convertTableSpec(name, spec, self._metadata, prefixes=['TEMPORARY'], 

934 schema=sqlalchemy.schema.BLANK_SCHEMA) 

935 if table.key in self._tempTables: 

936 if table.key != name: 

937 raise ValueError(f"A temporary table with name {name} (transformed to {table.key} by " 

938 f"Database) already exists.") 

939 for foreignKeySpec in spec.foreignKeys: 

940 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

941 table.create(self._connection) 

942 self._tempTables.add(table.key) 

943 return table 

944 

945 def dropTemporaryTable(self, table: sqlalchemy.schema.Table) -> None: 

946 """Drop a temporary table. 

947 

948 Parameters 

949 ---------- 

950 table : `sqlalchemy.schema.Table` 

951 A SQLAlchemy object returned by a previous call to 

952 `makeTemporaryTable`. 

953 """ 

954 if table.key in self._tempTables: 

955 table.drop(self._connection) 

956 self._tempTables.remove(table.key) 

957 else: 

958 raise TypeError(f"Table {table.key} was not created by makeTemporaryTable.") 

959 

960 @classmethod 

961 def getTimespanRepresentation(cls) -> Type[DatabaseTimespanRepresentation]: 

962 """Return a `type` that encapsulates the way `Timespan` objects are 

963 recommended to be stored in this database. 

964 

965 `Database` does not automatically use the return type of this method 

966 anywhere else; calling code is responsible for making sure that DDL 

967 and queries are consistent with it. 

968 

969 Returns 

970 ------- 

971 tsRepr : `type` (`DatabaseTimespanRepresention` subclass) 

972 A type that encapsultes the way `Timespan` objects should be 

973 stored in this database. 

974 

975 Notes 

976 ----- 

977 There are two big reasons we've decided to keep timespan-mangling logic 

978 outside the `Database` implementations, even though the choice of 

979 representation is ultimately up to a `Database` implementation: 

980 

981 - Timespans appear in relatively few tables and queries in our 

982 typical usage, and the code that operates on them is already aware 

983 that it is working with timespans. In contrast, a 

984 timespan-representation-aware implementation of, say, `insert`, 

985 would need to have extra logic to identify when timespan-mangling 

986 needed to occur, which would usually be useless overhead. 

987 

988 - SQLAlchemy's rich SELECT query expression system has no way to wrap 

989 multiple columns in a single expression object (the ORM does, but 

990 we are not using the ORM). So we would have to wrap _much_ more of 

991 that code in our own interfaces to encapsulate timespan 

992 representations there. 

993 """ 

994 return DatabaseTimespanRepresentation.Compound 

995 

996 def sync(self, table: sqlalchemy.schema.Table, *, 

997 keys: Dict[str, Any], 

998 compared: Optional[Dict[str, Any]] = None, 

999 extra: Optional[Dict[str, Any]] = None, 

1000 returning: Optional[Sequence[str]] = None, 

1001 ) -> Tuple[Optional[Dict[str, Any]], bool]: 

1002 """Insert into a table as necessary to ensure database contains 

1003 values equivalent to the given ones. 

1004 

1005 Parameters 

1006 ---------- 

1007 table : `sqlalchemy.schema.Table` 

1008 Table to be queried and possibly inserted into. 

1009 keys : `dict` 

1010 Column name-value pairs used to search for an existing row; must 

1011 be a combination that can be used to select a single row if one 

1012 exists. If such a row does not exist, these values are used in 

1013 the insert. 

1014 compared : `dict`, optional 

1015 Column name-value pairs that are compared to those in any existing 

1016 row. If such a row does not exist, these rows are used in the 

1017 insert. 

1018 extra : `dict`, optional 

1019 Column name-value pairs that are ignored if a matching row exists, 

1020 but used in an insert if one is necessary. 

1021 returning : `~collections.abc.Sequence` of `str`, optional 

1022 The names of columns whose values should be returned. 

1023 

1024 Returns 

1025 ------- 

1026 row : `dict`, optional 

1027 The value of the fields indicated by ``returning``, or `None` if 

1028 ``returning`` is `None`. 

1029 inserted : `bool` 

1030 If `True`, a new row was inserted. 

1031 

1032 Raises 

1033 ------ 

1034 DatabaseConflictError 

1035 Raised if the values in ``compared`` do not match the values in the 

1036 database. 

1037 ReadOnlyDatabaseError 

1038 Raised if `isWriteable` returns `False`, and no matching record 

1039 already exists. 

1040 

1041 Notes 

1042 ----- 

1043 May be used inside transaction contexts, so implementations may not 

1044 perform operations that interrupt transactions. 

1045 

1046 It may be called on read-only databases if and only if the matching row 

1047 does in fact already exist. 

1048 """ 

1049 

1050 def check() -> Tuple[int, Optional[List[str]], Optional[List]]: 

1051 """Query for a row that matches the ``key`` argument, and compare 

1052 to what was given by the caller. 

1053 

1054 Returns 

1055 ------- 

1056 n : `int` 

1057 Number of matching rows. ``n != 1`` is always an error, but 

1058 it's a different kind of error depending on where `check` is 

1059 being called. 

1060 bad : `list` of `str`, or `None` 

1061 The subset of the keys of ``compared`` for which the existing 

1062 values did not match the given one. Once again, ``not bad`` 

1063 is always an error, but a different kind on context. `None` 

1064 if ``n != 1`` 

1065 result : `list` or `None` 

1066 Results in the database that correspond to the columns given 

1067 in ``returning``, or `None` if ``returning is None``. 

1068 """ 

1069 toSelect: Set[str] = set() 

1070 if compared is not None: 

1071 toSelect.update(compared.keys()) 

1072 if returning is not None: 

1073 toSelect.update(returning) 

1074 if not toSelect: 

1075 # Need to select some column, even if we just want to see 

1076 # how many rows we get back. 

1077 toSelect.add(next(iter(keys.keys()))) 

1078 selectSql = sqlalchemy.sql.select( 

1079 [table.columns[k].label(k) for k in toSelect] 

1080 ).select_from(table).where( 

1081 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

1082 ) 

1083 fetched = list(self._connection.execute(selectSql).fetchall()) 

1084 if len(fetched) != 1: 

1085 return len(fetched), None, None 

1086 existing = fetched[0] 

1087 if compared is not None: 

1088 

1089 def safeNotEqual(a: Any, b: Any) -> bool: 

1090 if isinstance(a, astropy.time.Time): 

1091 return not time_utils.times_equal(a, b) 

1092 return a != b 

1093 

1094 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}" 

1095 for k, v in compared.items() 

1096 if safeNotEqual(existing[k], v)] 

1097 else: 

1098 inconsistencies = [] 

1099 if returning is not None: 

1100 toReturn: Optional[list] = [existing[k] for k in returning] 

1101 else: 

1102 toReturn = None 

1103 return 1, inconsistencies, toReturn 

1104 

1105 if self.isTableWriteable(table): 

1106 # Try an insert first, but allow it to fail (in only specific 

1107 # ways). 

1108 row = keys.copy() 

1109 if compared is not None: 

1110 row.update(compared) 

1111 if extra is not None: 

1112 row.update(extra) 

1113 with self.transaction(lock=[table]): 

1114 inserted = bool(self.ensure(table, row)) 

1115 # Need to perform check() for this branch inside the 

1116 # transaction, so we roll back an insert that didn't do 

1117 # what we expected. That limits the extent to which we 

1118 # can reduce duplication between this block and the other 

1119 # ones that perform similar logic. 

1120 n, bad, result = check() 

1121 if n < 1: 

1122 raise RuntimeError( 

1123 "Necessary insertion in sync did not seem to affect table. This is a bug." 

1124 ) 

1125 elif n > 1: 

1126 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

1127 f"unique constraint for table {table.name}.") 

1128 elif bad: 

1129 if inserted: 

1130 raise RuntimeError( 

1131 f"Conflict ({bad}) in sync after successful insert; this is " 

1132 "possible if the same table is being updated by a concurrent " 

1133 "process that isn't using sync, but it may also be a bug in " 

1134 "daf_butler." 

1135 ) 

1136 else: 

1137 raise DatabaseConflictError( 

1138 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1139 ) 

1140 else: 

1141 # Database is not writeable; just see if the row exists. 

1142 n, bad, result = check() 

1143 if n < 1: 

1144 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

1145 elif n > 1: 

1146 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

1147 elif bad: 

1148 raise DatabaseConflictError( 

1149 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1150 ) 

1151 inserted = False 

1152 if returning is None: 

1153 return None, inserted 

1154 else: 

1155 assert result is not None 

1156 return {k: v for k, v in zip(returning, result)}, inserted 

1157 

1158 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

1159 select: Optional[sqlalchemy.sql.Select] = None, 

1160 names: Optional[Iterable[str]] = None, 

1161 ) -> Optional[List[int]]: 

1162 """Insert one or more rows into a table, optionally returning 

1163 autoincrement primary key values. 

1164 

1165 Parameters 

1166 ---------- 

1167 table : `sqlalchemy.schema.Table` 

1168 Table rows should be inserted into. 

1169 returnIds: `bool` 

1170 If `True` (`False` is default), return the values of the table's 

1171 autoincrement primary key field (which much exist). 

1172 select : `sqlalchemy.sql.Select`, optional 

1173 A SELECT query expression to insert rows from. Cannot be provided 

1174 with either ``rows`` or ``returnIds=True``. 

1175 names : `Iterable` [ `str` ], optional 

1176 Names of columns in ``table`` to be populated, ordered to match the 

1177 columns returned by ``select``. Ignored if ``select`` is `None`. 

1178 If not provided, the columns returned by ``select`` must be named 

1179 to match the desired columns of ``table``. 

1180 *rows 

1181 Positional arguments are the rows to be inserted, as dictionaries 

1182 mapping column name to value. The keys in all dictionaries must 

1183 be the same. 

1184 

1185 Returns 

1186 ------- 

1187 ids : `None`, or `list` of `int` 

1188 If ``returnIds`` is `True`, a `list` containing the inserted 

1189 values for the table's autoincrement primary key. 

1190 

1191 Raises 

1192 ------ 

1193 ReadOnlyDatabaseError 

1194 Raised if `isWriteable` returns `False` when this method is called. 

1195 

1196 Notes 

1197 ----- 

1198 The default implementation uses bulk insert syntax when ``returnIds`` 

1199 is `False`, and a loop over single-row insert operations when it is 

1200 `True`. 

1201 

1202 Derived classes should reimplement when they can provide a more 

1203 efficient implementation (especially for the latter case). 

1204 

1205 May be used inside transaction contexts, so implementations may not 

1206 perform operations that interrupt transactions. 

1207 """ 

1208 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.") 

1209 if select is not None and (rows or returnIds): 

1210 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.") 

1211 if not rows and select is None: 

1212 if returnIds: 

1213 return [] 

1214 else: 

1215 return None 

1216 if not returnIds: 

1217 if select is not None: 

1218 if names is None: 

1219 names = select.columns.keys() 

1220 self._connection.execute(table.insert().from_select(names, select)) 

1221 else: 

1222 self._connection.execute(table.insert(), *rows) 

1223 return None 

1224 else: 

1225 sql = table.insert() 

1226 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

1227 

1228 @abstractmethod 

1229 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None: 

1230 """Insert one or more rows into a table, replacing any existing rows 

1231 for which insertion of a new row would violate the primary key 

1232 constraint. 

1233 

1234 Parameters 

1235 ---------- 

1236 table : `sqlalchemy.schema.Table` 

1237 Table rows should be inserted into. 

1238 *rows 

1239 Positional arguments are the rows to be inserted, as dictionaries 

1240 mapping column name to value. The keys in all dictionaries must 

1241 be the same. 

1242 

1243 Raises 

1244 ------ 

1245 ReadOnlyDatabaseError 

1246 Raised if `isWriteable` returns `False` when this method is called. 

1247 

1248 Notes 

1249 ----- 

1250 May be used inside transaction contexts, so implementations may not 

1251 perform operations that interrupt transactions. 

1252 

1253 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

1254 exception when a constraint other than the primary key would be 

1255 violated. 

1256 

1257 Implementations are not required to support `replace` on tables 

1258 with autoincrement keys. 

1259 """ 

1260 raise NotImplementedError() 

1261 

1262 @abstractmethod 

1263 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict) -> int: 

1264 """Insert one or more rows into a table, skipping any rows for which 

1265 insertion would violate any constraint. 

1266 

1267 Parameters 

1268 ---------- 

1269 table : `sqlalchemy.schema.Table` 

1270 Table rows should be inserted into. 

1271 *rows 

1272 Positional arguments are the rows to be inserted, as dictionaries 

1273 mapping column name to value. The keys in all dictionaries must 

1274 be the same. 

1275 

1276 Returns 

1277 ------- 

1278 count : `int` 

1279 The number of rows actually inserted. 

1280 

1281 Raises 

1282 ------ 

1283 ReadOnlyDatabaseError 

1284 Raised if `isWriteable` returns `False` when this method is called. 

1285 This is raised even if the operation would do nothing even on a 

1286 writeable database. 

1287 

1288 Notes 

1289 ----- 

1290 May be used inside transaction contexts, so implementations may not 

1291 perform operations that interrupt transactions. 

1292 

1293 Implementations are not required to support `ensure` on tables 

1294 with autoincrement keys. 

1295 """ 

1296 raise NotImplementedError() 

1297 

1298 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

1299 """Delete one or more rows from a table. 

1300 

1301 Parameters 

1302 ---------- 

1303 table : `sqlalchemy.schema.Table` 

1304 Table that rows should be deleted from. 

1305 columns: `~collections.abc.Iterable` of `str` 

1306 The names of columns that will be used to constrain the rows to 

1307 be deleted; these will be combined via ``AND`` to form the 

1308 ``WHERE`` clause of the delete query. 

1309 *rows 

1310 Positional arguments are the keys of rows to be deleted, as 

1311 dictionaries mapping column name to value. The keys in all 

1312 dictionaries must exactly the names in ``columns``. 

1313 

1314 Returns 

1315 ------- 

1316 count : `int` 

1317 Number of rows deleted. 

1318 

1319 Raises 

1320 ------ 

1321 ReadOnlyDatabaseError 

1322 Raised if `isWriteable` returns `False` when this method is called. 

1323 

1324 Notes 

1325 ----- 

1326 May be used inside transaction contexts, so implementations may not 

1327 perform operations that interrupt transactions. 

1328 

1329 The default implementation should be sufficient for most derived 

1330 classes. 

1331 """ 

1332 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.") 

1333 if columns and not rows: 

1334 # If there are no columns, this operation is supposed to delete 

1335 # everything (so we proceed as usual). But if there are columns, 

1336 # but no rows, it was a constrained bulk operation where the 

1337 # constraint is that no rows match, and we should short-circuit 

1338 # while reporting that no rows were affected. 

1339 return 0 

1340 sql = table.delete() 

1341 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

1342 if whereTerms: 

1343 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

1344 return self._connection.execute(sql, *rows).rowcount 

1345 

1346 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

1347 """Update one or more rows in a table. 

1348 

1349 Parameters 

1350 ---------- 

1351 table : `sqlalchemy.schema.Table` 

1352 Table containing the rows to be updated. 

1353 where : `dict` [`str`, `str`] 

1354 A mapping from the names of columns that will be used to search for 

1355 existing rows to the keys that will hold these values in the 

1356 ``rows`` dictionaries. Note that these may not be the same due to 

1357 SQLAlchemy limitations. 

1358 *rows 

1359 Positional arguments are the rows to be updated. The keys in all 

1360 dictionaries must be the same, and may correspond to either a 

1361 value in the ``where`` dictionary or the name of a column to be 

1362 updated. 

1363 

1364 Returns 

1365 ------- 

1366 count : `int` 

1367 Number of rows matched (regardless of whether the update actually 

1368 modified them). 

1369 

1370 Raises 

1371 ------ 

1372 ReadOnlyDatabaseError 

1373 Raised if `isWriteable` returns `False` when this method is called. 

1374 

1375 Notes 

1376 ----- 

1377 May be used inside transaction contexts, so implementations may not 

1378 perform operations that interrupt transactions. 

1379 

1380 The default implementation should be sufficient for most derived 

1381 classes. 

1382 """ 

1383 self.assertTableWriteable(table, f"Cannot update read-only table {table}.") 

1384 if not rows: 

1385 return 0 

1386 sql = table.update().where( 

1387 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1388 ) 

1389 return self._connection.execute(sql, *rows).rowcount 

1390 

1391 def query(self, sql: sqlalchemy.sql.FromClause, 

1392 *args: Any, **kwds: Any) -> sqlalchemy.engine.ResultProxy: 

1393 """Run a SELECT query against the database. 

1394 

1395 Parameters 

1396 ---------- 

1397 sql : `sqlalchemy.sql.FromClause` 

1398 A SQLAlchemy representation of a ``SELECT`` query. 

1399 *args 

1400 Additional positional arguments are forwarded to 

1401 `sqlalchemy.engine.Connection.execute`. 

1402 **kwds 

1403 Additional keyword arguments are forwarded to 

1404 `sqlalchemy.engine.Connection.execute`. 

1405 

1406 Returns 

1407 ------- 

1408 result : `sqlalchemy.engine.ResultProxy` 

1409 Query results. 

1410 

1411 Notes 

1412 ----- 

1413 The default implementation should be sufficient for most derived 

1414 classes. 

1415 """ 

1416 # TODO: should we guard against non-SELECT queries here? 

1417 return self._connection.execute(sql, *args, **kwds) 

1418 

1419 origin: int 

1420 """An integer ID that should be used as the default for any datasets, 

1421 quanta, or other entities that use a (autoincrement, origin) compound 

1422 primary key (`int`). 

1423 """ 

1424 

1425 namespace: Optional[str] 

1426 """The schema or namespace this database instance is associated with 

1427 (`str` or `None`). 

1428 """