Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "SchemaAlreadyDefinedError", 

28 "StaticTablesContext", 

29] 

30 

31from abc import ABC, abstractmethod 

32from contextlib import contextmanager 

33from typing import ( 

34 Any, 

35 Callable, 

36 Dict, 

37 Iterable, 

38 Iterator, 

39 List, 

40 Optional, 

41 Sequence, 

42 Set, 

43 Tuple, 

44 Type, 

45 Union, 

46) 

47import uuid 

48import warnings 

49 

50import astropy.time 

51import sqlalchemy 

52 

53from ...core import TimespanDatabaseRepresentation, ddl, time_utils 

54from .._exceptions import ConflictingDefinitionError 

55 

56_IN_SAVEPOINT_TRANSACTION = "IN_SAVEPOINT_TRANSACTION" 

57 

58 

59def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None: 

60 """Test that the definition of a table in a `ddl.TableSpec` and from 

61 database introspection are consistent. 

62 

63 Parameters 

64 ---------- 

65 name : `str` 

66 Name of the table (only used in error messages). 

67 spec : `ddl.TableSpec` 

68 Specification of the table. 

69 inspection : `dict` 

70 Dictionary returned by 

71 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

72 

73 Raises 

74 ------ 

75 DatabaseConflictError 

76 Raised if the definitions are inconsistent. 

77 """ 

78 columnNames = [c["name"] for c in inspection] 

79 if spec.fields.names != set(columnNames): 

80 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

81 f"specification has columns {list(spec.fields.names)}, while the " 

82 f"table in the database has {columnNames}.") 

83 

84 

85class ReadOnlyDatabaseError(RuntimeError): 

86 """Exception raised when a write operation is called on a read-only 

87 `Database`. 

88 """ 

89 

90 

91class DatabaseConflictError(ConflictingDefinitionError): 

92 """Exception raised when database content (row values or schema entities) 

93 are inconsistent with what this client expects. 

94 """ 

95 

96 

97class SchemaAlreadyDefinedError(RuntimeError): 

98 """Exception raised when trying to initialize database schema when some 

99 tables already exist. 

100 """ 

101 

102 

103class StaticTablesContext: 

104 """Helper class used to declare the static schema for a registry layer 

105 in a database. 

106 

107 An instance of this class is returned by `Database.declareStaticTables`, 

108 which should be the only way it should be constructed. 

109 """ 

110 

111 def __init__(self, db: Database): 

112 self._db = db 

113 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = [] 

114 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

115 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

116 self._initializers: List[Callable[[Database], None]] = [] 

117 

118 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

119 """Add a new table to the schema, returning its sqlalchemy 

120 representation. 

121 

122 The new table may not actually be created until the end of the 

123 context created by `Database.declareStaticTables`, allowing tables 

124 to be declared in any order even in the presence of foreign key 

125 relationships. 

126 """ 

127 name = self._db._mangleTableName(name) 

128 if name in self._tableNames: 

129 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

130 schema=self._db.namespace)) 

131 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

132 for foreignKeySpec in spec.foreignKeys: 

133 self._foreignKeys.append( 

134 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

135 ) 

136 return table 

137 

138 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

139 """Add a named tuple of tables to the schema, returning their 

140 SQLAlchemy representations in a named tuple of the same type. 

141 

142 The new tables may not actually be created until the end of the 

143 context created by `Database.declareStaticTables`, allowing tables 

144 to be declared in any order even in the presence of foreign key 

145 relationships. 

146 

147 Notes 

148 ----- 

149 ``specs`` *must* be an instance of a type created by 

150 `collections.namedtuple`, not just regular tuple, and the returned 

151 object is guaranteed to be the same. Because `~collections.namedtuple` 

152 is just a factory for `type` objects, not an actual type itself, 

153 we cannot represent this with type annotations. 

154 """ 

155 return specs._make(self.addTable(name, spec) # type: ignore 

156 for name, spec in zip(specs._fields, specs)) # type: ignore 

157 

158 def addInitializer(self, initializer: Callable[[Database], None]) -> None: 

159 """Add a method that does one-time initialization of a database. 

160 

161 Initialization can mean anything that changes state of a database 

162 and needs to be done exactly once after database schema was created. 

163 An example for that could be population of schema attributes. 

164 

165 Parameters 

166 ---------- 

167 initializer : callable 

168 Method of a single argument which is a `Database` instance. 

169 """ 

170 self._initializers.append(initializer) 

171 

172 

173class Database(ABC): 

174 """An abstract interface that represents a particular database engine's 

175 representation of a single schema/namespace/database. 

176 

177 Parameters 

178 ---------- 

179 origin : `int` 

180 An integer ID that should be used as the default for any datasets, 

181 quanta, or other entities that use a (autoincrement, origin) compound 

182 primary key. 

183 connection : `sqlalchemy.engine.Connection` 

184 The SQLAlchemy connection this `Database` wraps. 

185 namespace : `str`, optional 

186 Name of the schema or namespace this instance is associated with. 

187 This is passed as the ``schema`` argument when constructing a 

188 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

189 avoid confusion between "schema means namespace" and "schema means 

190 table definitions". 

191 

192 Notes 

193 ----- 

194 `Database` requires all write operations to go through its special named 

195 methods. Our write patterns are sufficiently simple that we don't really 

196 need the full flexibility of SQL insert/update/delete syntax, and we need 

197 non-standard (but common) functionality in these operations sufficiently 

198 often that it seems worthwhile to provide our own generic API. 

199 

200 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

201 their SQLAlchemy representation) to be run, as we expect these to require 

202 significantly more sophistication while still being limited to standard 

203 SQL. 

204 

205 `Database` itself has several underscore-prefixed attributes: 

206 

207 - ``_connection``: SQLAlchemy object representing the connection. 

208 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

209 the tables and other schema entities. 

210 

211 These are considered protected (derived classes may access them, but other 

212 code should not), and read-only, aside from executing SQL via 

213 ``_connection``. 

214 """ 

215 

216 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection, 

217 namespace: Optional[str] = None): 

218 self.origin = origin 

219 self.namespace = namespace 

220 self._connection = connection 

221 self._metadata: Optional[sqlalchemy.schema.MetaData] = None 

222 self._tempTables: Set[str] = set() 

223 

224 def __repr__(self) -> str: 

225 # Rather than try to reproduce all the parameters used to create 

226 # the object, instead report the more useful information of the 

227 # connection URL. 

228 uri = str(self._connection.engine.url) 

229 if self.namespace: 

230 uri += f"#{self.namespace}" 

231 return f'{type(self).__name__}("{uri}")' 

232 

233 @classmethod 

234 def makeDefaultUri(cls, root: str) -> Optional[str]: 

235 """Create a default connection URI appropriate for the given root 

236 directory, or `None` if there can be no such default. 

237 """ 

238 return None 

239 

240 @classmethod 

241 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

242 writeable: bool = True) -> Database: 

243 """Construct a database from a SQLAlchemy URI. 

244 

245 Parameters 

246 ---------- 

247 uri : `str` 

248 A SQLAlchemy URI connection string. 

249 origin : `int` 

250 An integer ID that should be used as the default for any datasets, 

251 quanta, or other entities that use a (autoincrement, origin) 

252 compound primary key. 

253 namespace : `str`, optional 

254 A database namespace (i.e. schema) the new instance should be 

255 associated with. If `None` (default), the namespace (if any) is 

256 inferred from the URI. 

257 writeable : `bool`, optional 

258 If `True`, allow write operations on the database, including 

259 ``CREATE TABLE``. 

260 

261 Returns 

262 ------- 

263 db : `Database` 

264 A new `Database` instance. 

265 """ 

266 return cls.fromConnection(cls.connect(uri, writeable=writeable), 

267 origin=origin, 

268 namespace=namespace, 

269 writeable=writeable) 

270 

271 @classmethod 

272 @abstractmethod 

273 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection: 

274 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI. 

275 

276 Parameters 

277 ---------- 

278 uri : `str` 

279 A SQLAlchemy URI connection string. 

280 origin : `int` 

281 An integer ID that should be used as the default for any datasets, 

282 quanta, or other entities that use a (autoincrement, origin) 

283 compound primary key. 

284 writeable : `bool`, optional 

285 If `True`, allow write operations on the database, including 

286 ``CREATE TABLE``. 

287 

288 Returns 

289 ------- 

290 connection : `sqlalchemy.engine.Connection` 

291 A database connection. 

292 

293 Notes 

294 ----- 

295 Subclasses that support other ways to connect to a database are 

296 encouraged to add optional arguments to their implementation of this 

297 method, as long as they maintain compatibility with the base class 

298 call signature. 

299 """ 

300 raise NotImplementedError() 

301 

302 @classmethod 

303 @abstractmethod 

304 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int, 

305 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

306 """Create a new `Database` from an existing 

307 `sqlalchemy.engine.Connection`. 

308 

309 Parameters 

310 ---------- 

311 connection : `sqllachemy.engine.Connection` 

312 The connection for the the database. May be shared between 

313 `Database` instances. 

314 origin : `int` 

315 An integer ID that should be used as the default for any datasets, 

316 quanta, or other entities that use a (autoincrement, origin) 

317 compound primary key. 

318 namespace : `str`, optional 

319 A different database namespace (i.e. schema) the new instance 

320 should be associated with. If `None` (default), the namespace 

321 (if any) is inferred from the connection. 

322 writeable : `bool`, optional 

323 If `True`, allow write operations on the database, including 

324 ``CREATE TABLE``. 

325 

326 Returns 

327 ------- 

328 db : `Database` 

329 A new `Database` instance. 

330 

331 Notes 

332 ----- 

333 This method allows different `Database` instances to share the same 

334 connection, which is desirable when they represent different namespaces 

335 can be queried together. This also ties their transaction state, 

336 however; starting a transaction in any database automatically starts 

337 on in all other databases. 

338 """ 

339 raise NotImplementedError() 

340 

341 @contextmanager 

342 def transaction(self, *, interrupting: bool = False, savepoint: bool = False, 

343 lock: Iterable[sqlalchemy.schema.Table] = ()) -> Iterator: 

344 """Return a context manager that represents a transaction. 

345 

346 Parameters 

347 ---------- 

348 interrupting : `bool`, optional 

349 If `True` (`False` is default), this transaction block may not be 

350 nested without an outer one, and attempting to do so is a logic 

351 (i.e. assertion) error. 

352 savepoint : `bool`, optional 

353 If `True` (`False` is default), create a `SAVEPOINT`, allowing 

354 exceptions raised by the database (e.g. due to constraint 

355 violations) during this transaction's context to be caught outside 

356 it without also rolling back all operations in an outer transaction 

357 block. If `False`, transactions may still be nested, but a 

358 rollback may be generated at any level and affects all levels, and 

359 commits are deferred until the outermost block completes. If any 

360 outer transaction block was created with ``savepoint=True``, all 

361 inner blocks will be as well (regardless of the actual value 

362 passed). This has no effect if this is the outermost transaction. 

363 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

364 A list of tables to lock for the duration of this transaction. 

365 These locks are guaranteed to prevent concurrent writes and allow 

366 this transaction (only) to acquire the same locks (others should 

367 block), but only prevent concurrent reads if the database engine 

368 requires that in order to block concurrent writes. 

369 

370 Notes 

371 ----- 

372 All transactions on a connection managed by one or more `Database` 

373 instances _must_ go through this method, or transaction state will not 

374 be correctly managed. 

375 """ 

376 assert not (interrupting and self._connection.in_transaction()), ( 

377 "Logic error in transaction nesting: an operation that would " 

378 "interrupt the active transaction context has been requested." 

379 ) 

380 # We remember whether we are already in a SAVEPOINT transaction via the 

381 # connection object's 'info' dict, which is explicitly for user 

382 # information like this. This is safer than a regular `Database` 

383 # instance attribute, because it guards against multiple `Database` 

384 # instances sharing the same connection. The need to use our own flag 

385 # here to track whether we're in a nested transaction should go away in 

386 # SQLAlchemy 1.4, which seems to have a 

387 # `Connection.in_nested_transaction()` method. 

388 savepoint = savepoint or self._connection.info.get(_IN_SAVEPOINT_TRANSACTION, False) 

389 self._connection.info[_IN_SAVEPOINT_TRANSACTION] = savepoint 

390 if self._connection.in_transaction() and savepoint: 

391 trans = self._connection.begin_nested() 

392 else: 

393 # Use a regular (non-savepoint) transaction always for the 

394 # outermost context, as well as when a savepoint was not requested. 

395 trans = self._connection.begin() 

396 self._lockTables(lock) 

397 try: 

398 yield 

399 trans.commit() 

400 except BaseException: 

401 trans.rollback() 

402 raise 

403 finally: 

404 if not self._connection.in_transaction(): 

405 self._connection.info.pop(_IN_SAVEPOINT_TRANSACTION, None) 

406 

407 @abstractmethod 

408 def _lockTables(self, tables: Iterable[sqlalchemy.schema.Table] = ()) -> None: 

409 """Acquire locks on the given tables. 

410 

411 This is an implementation hook for subclasses, called by `transaction`. 

412 It should not be called directly by other code. 

413 

414 Parameters 

415 ---------- 

416 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

417 A list of tables to lock for the duration of this transaction. 

418 These locks are guaranteed to prevent concurrent writes and allow 

419 this transaction (only) to acquire the same locks (others should 

420 block), but only prevent concurrent reads if the database engine 

421 requires that in order to block concurrent writes. 

422 """ 

423 raise NotImplementedError() 

424 

425 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool: 

426 """Check whether a table is writeable, either because the database 

427 connection is read-write or the table is a temporary table. 

428 

429 Parameters 

430 ---------- 

431 table : `sqlalchemy.schema.Table` 

432 SQLAlchemy table object to check. 

433 

434 Returns 

435 ------- 

436 writeable : `bool` 

437 Whether this table is writeable. 

438 """ 

439 return self.isWriteable() or table.key in self._tempTables 

440 

441 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None: 

442 """Raise if the given table is not writeable, either because the 

443 database connection is read-write or the table is a temporary table. 

444 

445 Parameters 

446 ---------- 

447 table : `sqlalchemy.schema.Table` 

448 SQLAlchemy table object to check. 

449 msg : `str`, optional 

450 If provided, raise `ReadOnlyDatabaseError` instead of returning 

451 `False`, with this message. 

452 """ 

453 if not self.isTableWriteable(table): 

454 raise ReadOnlyDatabaseError(msg) 

455 

456 @contextmanager 

457 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]: 

458 """Return a context manager in which the database's static DDL schema 

459 can be declared. 

460 

461 Parameters 

462 ---------- 

463 create : `bool` 

464 If `True`, attempt to create all tables at the end of the context. 

465 If `False`, they will be assumed to already exist. 

466 

467 Returns 

468 ------- 

469 schema : `StaticTablesContext` 

470 A helper object that is used to add new tables. 

471 

472 Raises 

473 ------ 

474 ReadOnlyDatabaseError 

475 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

476 and one or more declared tables do not already exist. 

477 

478 Examples 

479 -------- 

480 Given a `Database` instance ``db``:: 

481 

482 with db.declareStaticTables(create=True) as schema: 

483 schema.addTable("table1", TableSpec(...)) 

484 schema.addTable("table2", TableSpec(...)) 

485 

486 Notes 

487 ----- 

488 A database's static DDL schema must be declared before any dynamic 

489 tables are managed via calls to `ensureTableExists` or 

490 `getExistingTable`. The order in which static schema tables are added 

491 inside the context block is unimportant; they will automatically be 

492 sorted and added in an order consistent with their foreign key 

493 relationships. 

494 """ 

495 if create and not self.isWriteable(): 

496 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

497 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

498 try: 

499 context = StaticTablesContext(self) 

500 if create and context._tableNames: 

501 # Looks like database is already initalized, to avoid danger 

502 # of modifying/destroying valid schema we refuse to do 

503 # anything in this case 

504 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.") 

505 yield context 

506 for table, foreignKey in context._foreignKeys: 

507 table.append_constraint(foreignKey) 

508 if create: 

509 if self.namespace is not None: 

510 if self.namespace not in context._inspector.get_schema_names(): 

511 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

512 # In our tables we have columns that make use of sqlalchemy 

513 # Sequence objects. There is currently a bug in sqlalchemy that 

514 # causes a deprecation warning to be thrown on a property of 

515 # the Sequence object when the repr for the sequence is 

516 # created. Here a filter is used to catch these deprecation 

517 # warnings when tables are created. 

518 with warnings.catch_warnings(): 

519 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

520 self._metadata.create_all(self._connection) 

521 # call all initializer methods sequentially 

522 for init in context._initializers: 

523 init(self) 

524 except BaseException: 

525 self._metadata = None 

526 raise 

527 

528 @abstractmethod 

529 def isWriteable(self) -> bool: 

530 """Return `True` if this database can be modified by this client. 

531 """ 

532 raise NotImplementedError() 

533 

534 @abstractmethod 

535 def __str__(self) -> str: 

536 """Return a human-readable identifier for this `Database`, including 

537 any namespace or schema that identifies its names within a `Registry`. 

538 """ 

539 raise NotImplementedError() 

540 

541 @property 

542 def dialect(self) -> sqlalchemy.engine.Dialect: 

543 """The SQLAlchemy dialect for this database engine 

544 (`sqlalchemy.engine.Dialect`). 

545 """ 

546 return self._connection.dialect 

547 

548 def shrinkDatabaseEntityName(self, original: str) -> str: 

549 """Return a version of the given name that fits within this database 

550 engine's length limits for table, constraint, indexes, and sequence 

551 names. 

552 

553 Implementations should not assume that simple truncation is safe, 

554 because multiple long names often begin with the same prefix. 

555 

556 The default implementation simply returns the given name. 

557 

558 Parameters 

559 ---------- 

560 original : `str` 

561 The original name. 

562 

563 Returns 

564 ------- 

565 shrunk : `str` 

566 The new, possibly shortened name. 

567 """ 

568 return original 

569 

570 def expandDatabaseEntityName(self, shrunk: str) -> str: 

571 """Retrieve the original name for a database entity that was too long 

572 to fit within the database engine's limits. 

573 

574 Parameters 

575 ---------- 

576 original : `str` 

577 The original name. 

578 

579 Returns 

580 ------- 

581 shrunk : `str` 

582 The new, possibly shortened name. 

583 """ 

584 return shrunk 

585 

586 def _mangleTableName(self, name: str) -> str: 

587 """Map a logical, user-visible table name to the true table name used 

588 in the database. 

589 

590 The default implementation returns the given name unchanged. 

591 

592 Parameters 

593 ---------- 

594 name : `str` 

595 Input table name. Should not include a namespace (i.e. schema) 

596 prefix. 

597 

598 Returns 

599 ------- 

600 mangled : `str` 

601 Mangled version of the table name (still with no namespace prefix). 

602 

603 Notes 

604 ----- 

605 Reimplementations of this method must be idempotent - mangling an 

606 already-mangled name must have no effect. 

607 """ 

608 return name 

609 

610 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]: 

611 """Create constraints based on this spec. 

612 

613 Parameters 

614 ---------- 

615 table : `str` 

616 Name of the table this column is being added to. 

617 spec : `FieldSpec` 

618 Specification for the field to be added. 

619 

620 Returns 

621 ------- 

622 constraint : `list` of `sqlalchemy.CheckConstraint` 

623 Constraint added for this column. 

624 """ 

625 # By default we return no additional constraints 

626 return [] 

627 

628 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

629 **kwds: Any) -> sqlalchemy.schema.Column: 

630 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

631 

632 Parameters 

633 ---------- 

634 table : `str` 

635 Name of the table this column is being added to. 

636 spec : `FieldSpec` 

637 Specification for the field to be added. 

638 metadata : `sqlalchemy.MetaData` 

639 SQLAlchemy representation of the DDL schema this field's table is 

640 being added to. 

641 **kwds 

642 Additional keyword arguments to forward to the 

643 `sqlalchemy.schema.Column` constructor. This is provided to make 

644 it easier for derived classes to delegate to ``super()`` while 

645 making only minor changes. 

646 

647 Returns 

648 ------- 

649 column : `sqlalchemy.schema.Column` 

650 SQLAlchemy representation of the field. 

651 """ 

652 args = [spec.name, spec.getSizedColumnType()] 

653 if spec.autoincrement: 

654 # Generate a sequence to use for auto incrementing for databases 

655 # that do not support it natively. This will be ignored by 

656 # sqlalchemy for databases that do support it. 

657 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

658 metadata=metadata)) 

659 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

660 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

661 comment=spec.doc, server_default=spec.default, **kwds) 

662 

663 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

664 **kwds: Any) -> sqlalchemy.schema.ForeignKeyConstraint: 

665 """Convert a `ForeignKeySpec` to a 

666 `sqlalchemy.schema.ForeignKeyConstraint`. 

667 

668 Parameters 

669 ---------- 

670 table : `str` 

671 Name of the table this foreign key is being added to. 

672 spec : `ForeignKeySpec` 

673 Specification for the foreign key to be added. 

674 metadata : `sqlalchemy.MetaData` 

675 SQLAlchemy representation of the DDL schema this constraint is 

676 being added to. 

677 **kwds 

678 Additional keyword arguments to forward to the 

679 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

680 provided to make it easier for derived classes to delegate to 

681 ``super()`` while making only minor changes. 

682 

683 Returns 

684 ------- 

685 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

686 SQLAlchemy representation of the constraint. 

687 """ 

688 name = self.shrinkDatabaseEntityName( 

689 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

690 + list(spec.target) + list(spec.source)) 

691 ) 

692 return sqlalchemy.schema.ForeignKeyConstraint( 

693 spec.source, 

694 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

695 name=name, 

696 ondelete=spec.onDelete 

697 ) 

698 

699 def _convertExclusionConstraintSpec(self, table: str, 

700 spec: Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...], 

701 metadata: sqlalchemy.MetaData) -> sqlalchemy.schema.Constraint: 

702 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy 

703 constraint representation. 

704 

705 Parameters 

706 ---------- 

707 table : `str` 

708 Name of the table this constraint is being added to. 

709 spec : `tuple` [ `str` or `type` ] 

710 A tuple of `str` column names and the `type` object returned by 

711 `getTimespanRepresentation` (which must appear exactly once), 

712 indicating the order of the columns in the index used to back the 

713 constraint. 

714 metadata : `sqlalchemy.MetaData` 

715 SQLAlchemy representation of the DDL schema this constraint is 

716 being added to. 

717 

718 Returns 

719 ------- 

720 constraint : `sqlalchemy.schema.Constraint` 

721 SQLAlchemy representation of the constraint. 

722 

723 Raises 

724 ------ 

725 NotImplementedError 

726 Raised if this database does not support exclusion constraints. 

727 """ 

728 raise NotImplementedError(f"Database {self} does not support exclusion constraints.") 

729 

730 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

731 **kwds: Any) -> sqlalchemy.schema.Table: 

732 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

733 

734 Parameters 

735 ---------- 

736 spec : `TableSpec` 

737 Specification for the foreign key to be added. 

738 metadata : `sqlalchemy.MetaData` 

739 SQLAlchemy representation of the DDL schema this table is being 

740 added to. 

741 **kwds 

742 Additional keyword arguments to forward to the 

743 `sqlalchemy.schema.Table` constructor. This is provided to make it 

744 easier for derived classes to delegate to ``super()`` while making 

745 only minor changes. 

746 

747 Returns 

748 ------- 

749 table : `sqlalchemy.schema.Table` 

750 SQLAlchemy representation of the table. 

751 

752 Notes 

753 ----- 

754 This method does not handle ``spec.foreignKeys`` at all, in order to 

755 avoid circular dependencies. These are added by higher-level logic in 

756 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

757 """ 

758 name = self._mangleTableName(name) 

759 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

760 

761 # Add any column constraints 

762 for fieldSpec in spec.fields: 

763 args.extend(self._makeColumnConstraints(name, fieldSpec)) 

764 

765 # Track indexes added for primary key and unique constraints, to make 

766 # sure we don't add duplicate explicit or foreign key indexes for 

767 # those. 

768 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)} 

769 args.extend( 

770 sqlalchemy.schema.UniqueConstraint( 

771 *columns, 

772 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

773 ) 

774 for columns in spec.unique 

775 ) 

776 allIndexes.update(spec.unique) 

777 args.extend( 

778 sqlalchemy.schema.Index( 

779 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

780 *columns, 

781 unique=(columns in spec.unique) 

782 ) 

783 for columns in spec.indexes if columns not in allIndexes 

784 ) 

785 allIndexes.update(spec.indexes) 

786 args.extend( 

787 sqlalchemy.schema.Index( 

788 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)), 

789 *fk.source, 

790 ) 

791 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes 

792 ) 

793 

794 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion) 

795 

796 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

797 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

798 

799 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

800 """Ensure that a table with the given name and specification exists, 

801 creating it if necessary. 

802 

803 Parameters 

804 ---------- 

805 name : `str` 

806 Name of the table (not including namespace qualifiers). 

807 spec : `TableSpec` 

808 Specification for the table. This will be used when creating the 

809 table, and *may* be used when obtaining an existing table to check 

810 for consistency, but no such check is guaranteed. 

811 

812 Returns 

813 ------- 

814 table : `sqlalchemy.schema.Table` 

815 SQLAlchemy representation of the table. 

816 

817 Raises 

818 ------ 

819 ReadOnlyDatabaseError 

820 Raised if `isWriteable` returns `False`, and the table does not 

821 already exist. 

822 DatabaseConflictError 

823 Raised if the table exists but ``spec`` is inconsistent with its 

824 definition. 

825 

826 Notes 

827 ----- 

828 This method may not be called within transactions. It may be called on 

829 read-only databases if and only if the table does in fact already 

830 exist. 

831 

832 Subclasses may override this method, but usually should not need to. 

833 """ 

834 assert not self._connection.in_transaction(), "Table creation interrupts transactions." 

835 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

836 table = self.getExistingTable(name, spec) 

837 if table is not None: 

838 return table 

839 if not self.isWriteable(): 

840 raise ReadOnlyDatabaseError( 

841 f"Table {name} does not exist, and cannot be created " 

842 f"because database {self} is read-only." 

843 ) 

844 table = self._convertTableSpec(name, spec, self._metadata) 

845 for foreignKeySpec in spec.foreignKeys: 

846 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

847 table.create(self._connection) 

848 return table 

849 

850 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

851 """Obtain an existing table with the given name and specification. 

852 

853 Parameters 

854 ---------- 

855 name : `str` 

856 Name of the table (not including namespace qualifiers). 

857 spec : `TableSpec` 

858 Specification for the table. This will be used when creating the 

859 SQLAlchemy representation of the table, and it is used to 

860 check that the actual table in the database is consistent. 

861 

862 Returns 

863 ------- 

864 table : `sqlalchemy.schema.Table` or `None` 

865 SQLAlchemy representation of the table, or `None` if it does not 

866 exist. 

867 

868 Raises 

869 ------ 

870 DatabaseConflictError 

871 Raised if the table exists but ``spec`` is inconsistent with its 

872 definition. 

873 

874 Notes 

875 ----- 

876 This method can be called within transactions and never modifies the 

877 database. 

878 

879 Subclasses may override this method, but usually should not need to. 

880 """ 

881 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

882 name = self._mangleTableName(name) 

883 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

884 if table is not None: 

885 if spec.fields.names != set(table.columns.keys()): 

886 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

887 f"specification has columns {list(spec.fields.names)}, while " 

888 f"the previous definition has {list(table.columns.keys())}.") 

889 else: 

890 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

891 if name in inspector.get_table_names(schema=self.namespace): 

892 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

893 table = self._convertTableSpec(name, spec, self._metadata) 

894 for foreignKeySpec in spec.foreignKeys: 

895 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

896 return table 

897 return table 

898 

899 def makeTemporaryTable(self, spec: ddl.TableSpec, name: Optional[str] = None) -> sqlalchemy.schema.Table: 

900 """Create a temporary table. 

901 

902 Parameters 

903 ---------- 

904 spec : `TableSpec` 

905 Specification for the table. 

906 name : `str`, optional 

907 A unique (within this session/connetion) name for the table. 

908 Subclasses may override to modify the actual name used. If not 

909 provided, a unique name will be generated. 

910 

911 Returns 

912 ------- 

913 table : `sqlalchemy.schema.Table` 

914 SQLAlchemy representation of the table. 

915 

916 Notes 

917 ----- 

918 Temporary tables may be created, dropped, and written to even in 

919 read-only databases - at least according to the Python-level 

920 protections in the `Database` classes. Server permissions may say 

921 otherwise, but in that case they probably need to be modified to 

922 support the full range of expected read-only butler behavior. 

923 

924 Temporary table rows are guaranteed to be dropped when a connection is 

925 closed. `Database` implementations are permitted to allow the table to 

926 remain as long as this is transparent to the user (i.e. "creating" the 

927 temporary table in a new session should not be an error, even if it 

928 does nothing). 

929 

930 It may not be possible to use temporary tables within transactions with 

931 some database engines (or configurations thereof). 

932 """ 

933 if name is None: 

934 name = f"tmp_{uuid.uuid4().hex}" 

935 table = self._convertTableSpec(name, spec, self._metadata, prefixes=['TEMPORARY'], 

936 schema=sqlalchemy.schema.BLANK_SCHEMA) 

937 if table.key in self._tempTables: 

938 if table.key != name: 

939 raise ValueError(f"A temporary table with name {name} (transformed to {table.key} by " 

940 f"Database) already exists.") 

941 for foreignKeySpec in spec.foreignKeys: 

942 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

943 table.create(self._connection) 

944 self._tempTables.add(table.key) 

945 return table 

946 

947 def dropTemporaryTable(self, table: sqlalchemy.schema.Table) -> None: 

948 """Drop a temporary table. 

949 

950 Parameters 

951 ---------- 

952 table : `sqlalchemy.schema.Table` 

953 A SQLAlchemy object returned by a previous call to 

954 `makeTemporaryTable`. 

955 """ 

956 if table.key in self._tempTables: 

957 table.drop(self._connection) 

958 self._tempTables.remove(table.key) 

959 else: 

960 raise TypeError(f"Table {table.key} was not created by makeTemporaryTable.") 

961 

962 @classmethod 

963 def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]: 

964 """Return a `type` that encapsulates the way `Timespan` objects are 

965 recommended to be stored in this database. 

966 

967 `Database` does not automatically use the return type of this method 

968 anywhere else; calling code is responsible for making sure that DDL 

969 and queries are consistent with it. 

970 

971 Returns 

972 ------- 

973 tsRepr : `type` (`DatabaseTimespanRepresention` subclass) 

974 A type that encapsultes the way `Timespan` objects should be 

975 stored in this database. 

976 

977 Notes 

978 ----- 

979 There are two big reasons we've decided to keep timespan-mangling logic 

980 outside the `Database` implementations, even though the choice of 

981 representation is ultimately up to a `Database` implementation: 

982 

983 - Timespans appear in relatively few tables and queries in our 

984 typical usage, and the code that operates on them is already aware 

985 that it is working with timespans. In contrast, a 

986 timespan-representation-aware implementation of, say, `insert`, 

987 would need to have extra logic to identify when timespan-mangling 

988 needed to occur, which would usually be useless overhead. 

989 

990 - SQLAlchemy's rich SELECT query expression system has no way to wrap 

991 multiple columns in a single expression object (the ORM does, but 

992 we are not using the ORM). So we would have to wrap _much_ more of 

993 that code in our own interfaces to encapsulate timespan 

994 representations there. 

995 """ 

996 return TimespanDatabaseRepresentation.Compound 

997 

998 def sync(self, table: sqlalchemy.schema.Table, *, 

999 keys: Dict[str, Any], 

1000 compared: Optional[Dict[str, Any]] = None, 

1001 extra: Optional[Dict[str, Any]] = None, 

1002 returning: Optional[Sequence[str]] = None, 

1003 ) -> Tuple[Optional[Dict[str, Any]], bool]: 

1004 """Insert into a table as necessary to ensure database contains 

1005 values equivalent to the given ones. 

1006 

1007 Parameters 

1008 ---------- 

1009 table : `sqlalchemy.schema.Table` 

1010 Table to be queried and possibly inserted into. 

1011 keys : `dict` 

1012 Column name-value pairs used to search for an existing row; must 

1013 be a combination that can be used to select a single row if one 

1014 exists. If such a row does not exist, these values are used in 

1015 the insert. 

1016 compared : `dict`, optional 

1017 Column name-value pairs that are compared to those in any existing 

1018 row. If such a row does not exist, these rows are used in the 

1019 insert. 

1020 extra : `dict`, optional 

1021 Column name-value pairs that are ignored if a matching row exists, 

1022 but used in an insert if one is necessary. 

1023 returning : `~collections.abc.Sequence` of `str`, optional 

1024 The names of columns whose values should be returned. 

1025 

1026 Returns 

1027 ------- 

1028 row : `dict`, optional 

1029 The value of the fields indicated by ``returning``, or `None` if 

1030 ``returning`` is `None`. 

1031 inserted : `bool` 

1032 If `True`, a new row was inserted. 

1033 

1034 Raises 

1035 ------ 

1036 DatabaseConflictError 

1037 Raised if the values in ``compared`` do not match the values in the 

1038 database. 

1039 ReadOnlyDatabaseError 

1040 Raised if `isWriteable` returns `False`, and no matching record 

1041 already exists. 

1042 

1043 Notes 

1044 ----- 

1045 May be used inside transaction contexts, so implementations may not 

1046 perform operations that interrupt transactions. 

1047 

1048 It may be called on read-only databases if and only if the matching row 

1049 does in fact already exist. 

1050 """ 

1051 

1052 def check() -> Tuple[int, Optional[List[str]], Optional[List]]: 

1053 """Query for a row that matches the ``key`` argument, and compare 

1054 to what was given by the caller. 

1055 

1056 Returns 

1057 ------- 

1058 n : `int` 

1059 Number of matching rows. ``n != 1`` is always an error, but 

1060 it's a different kind of error depending on where `check` is 

1061 being called. 

1062 bad : `list` of `str`, or `None` 

1063 The subset of the keys of ``compared`` for which the existing 

1064 values did not match the given one. Once again, ``not bad`` 

1065 is always an error, but a different kind on context. `None` 

1066 if ``n != 1`` 

1067 result : `list` or `None` 

1068 Results in the database that correspond to the columns given 

1069 in ``returning``, or `None` if ``returning is None``. 

1070 """ 

1071 toSelect: Set[str] = set() 

1072 if compared is not None: 

1073 toSelect.update(compared.keys()) 

1074 if returning is not None: 

1075 toSelect.update(returning) 

1076 if not toSelect: 

1077 # Need to select some column, even if we just want to see 

1078 # how many rows we get back. 

1079 toSelect.add(next(iter(keys.keys()))) 

1080 selectSql = sqlalchemy.sql.select( 

1081 [table.columns[k].label(k) for k in toSelect] 

1082 ).select_from(table).where( 

1083 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

1084 ) 

1085 fetched = list(self._connection.execute(selectSql).fetchall()) 

1086 if len(fetched) != 1: 

1087 return len(fetched), None, None 

1088 existing = fetched[0] 

1089 if compared is not None: 

1090 

1091 def safeNotEqual(a: Any, b: Any) -> bool: 

1092 if isinstance(a, astropy.time.Time): 

1093 return not time_utils.times_equal(a, b) 

1094 return a != b 

1095 

1096 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}" 

1097 for k, v in compared.items() 

1098 if safeNotEqual(existing[k], v)] 

1099 else: 

1100 inconsistencies = [] 

1101 if returning is not None: 

1102 toReturn: Optional[list] = [existing[k] for k in returning] 

1103 else: 

1104 toReturn = None 

1105 return 1, inconsistencies, toReturn 

1106 

1107 if self.isTableWriteable(table): 

1108 # Try an insert first, but allow it to fail (in only specific 

1109 # ways). 

1110 row = keys.copy() 

1111 if compared is not None: 

1112 row.update(compared) 

1113 if extra is not None: 

1114 row.update(extra) 

1115 with self.transaction(lock=[table]): 

1116 inserted = bool(self.ensure(table, row)) 

1117 # Need to perform check() for this branch inside the 

1118 # transaction, so we roll back an insert that didn't do 

1119 # what we expected. That limits the extent to which we 

1120 # can reduce duplication between this block and the other 

1121 # ones that perform similar logic. 

1122 n, bad, result = check() 

1123 if n < 1: 

1124 raise RuntimeError( 

1125 "Necessary insertion in sync did not seem to affect table. This is a bug." 

1126 ) 

1127 elif n > 1: 

1128 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

1129 f"unique constraint for table {table.name}.") 

1130 elif bad: 

1131 if inserted: 

1132 raise RuntimeError( 

1133 f"Conflict ({bad}) in sync after successful insert; this is " 

1134 "possible if the same table is being updated by a concurrent " 

1135 "process that isn't using sync, but it may also be a bug in " 

1136 "daf_butler." 

1137 ) 

1138 else: 

1139 raise DatabaseConflictError( 

1140 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1141 ) 

1142 else: 

1143 # Database is not writeable; just see if the row exists. 

1144 n, bad, result = check() 

1145 if n < 1: 

1146 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

1147 elif n > 1: 

1148 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

1149 elif bad: 

1150 raise DatabaseConflictError( 

1151 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1152 ) 

1153 inserted = False 

1154 if returning is None: 

1155 return None, inserted 

1156 else: 

1157 assert result is not None 

1158 return {k: v for k, v in zip(returning, result)}, inserted 

1159 

1160 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

1161 select: Optional[sqlalchemy.sql.Select] = None, 

1162 names: Optional[Iterable[str]] = None, 

1163 ) -> Optional[List[int]]: 

1164 """Insert one or more rows into a table, optionally returning 

1165 autoincrement primary key values. 

1166 

1167 Parameters 

1168 ---------- 

1169 table : `sqlalchemy.schema.Table` 

1170 Table rows should be inserted into. 

1171 returnIds: `bool` 

1172 If `True` (`False` is default), return the values of the table's 

1173 autoincrement primary key field (which much exist). 

1174 select : `sqlalchemy.sql.Select`, optional 

1175 A SELECT query expression to insert rows from. Cannot be provided 

1176 with either ``rows`` or ``returnIds=True``. 

1177 names : `Iterable` [ `str` ], optional 

1178 Names of columns in ``table`` to be populated, ordered to match the 

1179 columns returned by ``select``. Ignored if ``select`` is `None`. 

1180 If not provided, the columns returned by ``select`` must be named 

1181 to match the desired columns of ``table``. 

1182 *rows 

1183 Positional arguments are the rows to be inserted, as dictionaries 

1184 mapping column name to value. The keys in all dictionaries must 

1185 be the same. 

1186 

1187 Returns 

1188 ------- 

1189 ids : `None`, or `list` of `int` 

1190 If ``returnIds`` is `True`, a `list` containing the inserted 

1191 values for the table's autoincrement primary key. 

1192 

1193 Raises 

1194 ------ 

1195 ReadOnlyDatabaseError 

1196 Raised if `isWriteable` returns `False` when this method is called. 

1197 

1198 Notes 

1199 ----- 

1200 The default implementation uses bulk insert syntax when ``returnIds`` 

1201 is `False`, and a loop over single-row insert operations when it is 

1202 `True`. 

1203 

1204 Derived classes should reimplement when they can provide a more 

1205 efficient implementation (especially for the latter case). 

1206 

1207 May be used inside transaction contexts, so implementations may not 

1208 perform operations that interrupt transactions. 

1209 """ 

1210 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.") 

1211 if select is not None and (rows or returnIds): 

1212 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.") 

1213 if not rows and select is None: 

1214 if returnIds: 

1215 return [] 

1216 else: 

1217 return None 

1218 if not returnIds: 

1219 if select is not None: 

1220 if names is None: 

1221 names = select.columns.keys() 

1222 self._connection.execute(table.insert().from_select(names, select)) 

1223 else: 

1224 self._connection.execute(table.insert(), *rows) 

1225 return None 

1226 else: 

1227 sql = table.insert() 

1228 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

1229 

1230 @abstractmethod 

1231 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None: 

1232 """Insert one or more rows into a table, replacing any existing rows 

1233 for which insertion of a new row would violate the primary key 

1234 constraint. 

1235 

1236 Parameters 

1237 ---------- 

1238 table : `sqlalchemy.schema.Table` 

1239 Table rows should be inserted into. 

1240 *rows 

1241 Positional arguments are the rows to be inserted, as dictionaries 

1242 mapping column name to value. The keys in all dictionaries must 

1243 be the same. 

1244 

1245 Raises 

1246 ------ 

1247 ReadOnlyDatabaseError 

1248 Raised if `isWriteable` returns `False` when this method is called. 

1249 

1250 Notes 

1251 ----- 

1252 May be used inside transaction contexts, so implementations may not 

1253 perform operations that interrupt transactions. 

1254 

1255 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

1256 exception when a constraint other than the primary key would be 

1257 violated. 

1258 

1259 Implementations are not required to support `replace` on tables 

1260 with autoincrement keys. 

1261 """ 

1262 raise NotImplementedError() 

1263 

1264 @abstractmethod 

1265 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict) -> int: 

1266 """Insert one or more rows into a table, skipping any rows for which 

1267 insertion would violate any constraint. 

1268 

1269 Parameters 

1270 ---------- 

1271 table : `sqlalchemy.schema.Table` 

1272 Table rows should be inserted into. 

1273 *rows 

1274 Positional arguments are the rows to be inserted, as dictionaries 

1275 mapping column name to value. The keys in all dictionaries must 

1276 be the same. 

1277 

1278 Returns 

1279 ------- 

1280 count : `int` 

1281 The number of rows actually inserted. 

1282 

1283 Raises 

1284 ------ 

1285 ReadOnlyDatabaseError 

1286 Raised if `isWriteable` returns `False` when this method is called. 

1287 This is raised even if the operation would do nothing even on a 

1288 writeable database. 

1289 

1290 Notes 

1291 ----- 

1292 May be used inside transaction contexts, so implementations may not 

1293 perform operations that interrupt transactions. 

1294 

1295 Implementations are not required to support `ensure` on tables 

1296 with autoincrement keys. 

1297 """ 

1298 raise NotImplementedError() 

1299 

1300 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

1301 """Delete one or more rows from a table. 

1302 

1303 Parameters 

1304 ---------- 

1305 table : `sqlalchemy.schema.Table` 

1306 Table that rows should be deleted from. 

1307 columns: `~collections.abc.Iterable` of `str` 

1308 The names of columns that will be used to constrain the rows to 

1309 be deleted; these will be combined via ``AND`` to form the 

1310 ``WHERE`` clause of the delete query. 

1311 *rows 

1312 Positional arguments are the keys of rows to be deleted, as 

1313 dictionaries mapping column name to value. The keys in all 

1314 dictionaries must exactly the names in ``columns``. 

1315 

1316 Returns 

1317 ------- 

1318 count : `int` 

1319 Number of rows deleted. 

1320 

1321 Raises 

1322 ------ 

1323 ReadOnlyDatabaseError 

1324 Raised if `isWriteable` returns `False` when this method is called. 

1325 

1326 Notes 

1327 ----- 

1328 May be used inside transaction contexts, so implementations may not 

1329 perform operations that interrupt transactions. 

1330 

1331 The default implementation should be sufficient for most derived 

1332 classes. 

1333 """ 

1334 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.") 

1335 if columns and not rows: 

1336 # If there are no columns, this operation is supposed to delete 

1337 # everything (so we proceed as usual). But if there are columns, 

1338 # but no rows, it was a constrained bulk operation where the 

1339 # constraint is that no rows match, and we should short-circuit 

1340 # while reporting that no rows were affected. 

1341 return 0 

1342 sql = table.delete() 

1343 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

1344 if whereTerms: 

1345 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

1346 return self._connection.execute(sql, *rows).rowcount 

1347 

1348 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

1349 """Update one or more rows in a table. 

1350 

1351 Parameters 

1352 ---------- 

1353 table : `sqlalchemy.schema.Table` 

1354 Table containing the rows to be updated. 

1355 where : `dict` [`str`, `str`] 

1356 A mapping from the names of columns that will be used to search for 

1357 existing rows to the keys that will hold these values in the 

1358 ``rows`` dictionaries. Note that these may not be the same due to 

1359 SQLAlchemy limitations. 

1360 *rows 

1361 Positional arguments are the rows to be updated. The keys in all 

1362 dictionaries must be the same, and may correspond to either a 

1363 value in the ``where`` dictionary or the name of a column to be 

1364 updated. 

1365 

1366 Returns 

1367 ------- 

1368 count : `int` 

1369 Number of rows matched (regardless of whether the update actually 

1370 modified them). 

1371 

1372 Raises 

1373 ------ 

1374 ReadOnlyDatabaseError 

1375 Raised if `isWriteable` returns `False` when this method is called. 

1376 

1377 Notes 

1378 ----- 

1379 May be used inside transaction contexts, so implementations may not 

1380 perform operations that interrupt transactions. 

1381 

1382 The default implementation should be sufficient for most derived 

1383 classes. 

1384 """ 

1385 self.assertTableWriteable(table, f"Cannot update read-only table {table}.") 

1386 if not rows: 

1387 return 0 

1388 sql = table.update().where( 

1389 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1390 ) 

1391 return self._connection.execute(sql, *rows).rowcount 

1392 

1393 def query(self, sql: sqlalchemy.sql.FromClause, 

1394 *args: Any, **kwds: Any) -> sqlalchemy.engine.ResultProxy: 

1395 """Run a SELECT query against the database. 

1396 

1397 Parameters 

1398 ---------- 

1399 sql : `sqlalchemy.sql.FromClause` 

1400 A SQLAlchemy representation of a ``SELECT`` query. 

1401 *args 

1402 Additional positional arguments are forwarded to 

1403 `sqlalchemy.engine.Connection.execute`. 

1404 **kwds 

1405 Additional keyword arguments are forwarded to 

1406 `sqlalchemy.engine.Connection.execute`. 

1407 

1408 Returns 

1409 ------- 

1410 result : `sqlalchemy.engine.ResultProxy` 

1411 Query results. 

1412 

1413 Notes 

1414 ----- 

1415 The default implementation should be sufficient for most derived 

1416 classes. 

1417 """ 

1418 # TODO: should we guard against non-SELECT queries here? 

1419 return self._connection.execute(sql, *args, **kwds) 

1420 

1421 origin: int 

1422 """An integer ID that should be used as the default for any datasets, 

1423 quanta, or other entities that use a (autoincrement, origin) compound 

1424 primary key (`int`). 

1425 """ 

1426 

1427 namespace: Optional[str] 

1428 """The schema or namespace this database instance is associated with 

1429 (`str` or `None`). 

1430 """