Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "StaticTablesContext", 

28] 

29 

30from abc import ABC, abstractmethod 

31from contextlib import contextmanager 

32from typing import ( 

33 Any, 

34 Dict, 

35 Iterable, 

36 List, 

37 Optional, 

38 Sequence, 

39 Tuple, 

40) 

41import warnings 

42 

43import sqlalchemy 

44 

45from ...core import ddl 

46 

47 

48def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: Dict[str, Any]): 

49 """Test that the definition of a table in a `ddl.TableSpec` and from 

50 database introspection are consistent. 

51 

52 Parameters 

53 ---------- 

54 name : `str` 

55 Name of the table (only used in error messages). 

56 spec : `ddl.TableSpec` 

57 Specification of the table. 

58 inspection : `dict` 

59 Dictionary returned by 

60 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

61 

62 Raises 

63 ------ 

64 DatabaseConflictError 

65 Raised if the definitions are inconsistent. 

66 """ 

67 columnNames = [c["name"] for c in inspection] 

68 if spec.fields.names != set(columnNames): 

69 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

70 f"specification has columns {list(spec.fields.names)}, while the " 

71 f"table in the database has {columnNames}.") 

72 

73 

74class ReadOnlyDatabaseError(RuntimeError): 

75 """Exception raised when a write operation is called on a read-only 

76 `Database`. 

77 """ 

78 

79 

80class DatabaseConflictError(RuntimeError): 

81 """Exception raised when database content (row values or schema entities) 

82 are inconsistent with what this client expects. 

83 """ 

84 

85 

86class StaticTablesContext: 

87 """Helper class used to declare the static schema for a registry layer 

88 in a database. 

89 

90 An instance of this class is returned by `Database.declareStaticTables`, 

91 which should be the only way it should be constructed. 

92 """ 

93 

94 def __init__(self, db: Database): 

95 self._db = db 

96 self._foreignKeys = [] 

97 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

98 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

99 

100 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

101 """Add a new table to the schema, returning its sqlalchemy 

102 representation. 

103 

104 The new table may not actually be created until the end of the 

105 context created by `Database.declareStaticTables`, allowing tables 

106 to be declared in any order even in the presence of foreign key 

107 relationships. 

108 """ 

109 name = self._db._mangleTableName(name) 

110 if name in self._tableNames: 

111 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

112 schema=self._db.namespace)) 

113 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

114 for foreignKeySpec in spec.foreignKeys: 

115 self._foreignKeys.append( 

116 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

117 ) 

118 return table 

119 

120 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

121 """Add a named tuple of tables to the schema, returning their 

122 SQLAlchemy representations in a named tuple of the same type. 

123 

124 The new tables may not actually be created until the end of the 

125 context created by `Database.declareStaticTables`, allowing tables 

126 to be declared in any order even in the presence of foreign key 

127 relationships. 

128 

129 Notes 

130 ----- 

131 ``specs`` *must* be an instance of a type created by 

132 `collections.namedtuple`, not just regular tuple, and the returned 

133 object is guaranteed to be the same. Because `~collections.namedtuple` 

134 is just a factory for `type` objects, not an actual type itself, 

135 we cannot represent this with type annotations. 

136 """ 

137 return specs._make(self.addTable(name, spec) for name, spec in zip(specs._fields, specs)) 

138 

139 

140class Database(ABC): 

141 """An abstract interface that represents a particular database engine's 

142 representation of a single schema/namespace/database. 

143 

144 Parameters 

145 ---------- 

146 origin : `int` 

147 An integer ID that should be used as the default for any datasets, 

148 quanta, or other entities that use a (autoincrement, origin) compound 

149 primary key. 

150 connection : `sqlalchemy.engine.Connection` 

151 The SQLAlchemy connection this `Database` wraps. 

152 namespace : `str`, optional 

153 Name of the schema or namespace this instance is associated with. 

154 This is passed as the ``schema`` argument when constructing a 

155 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

156 avoid confusion between "schema means namespace" and "schema means 

157 table definitions". 

158 

159 Notes 

160 ----- 

161 `Database` requires all write operations to go through its special named 

162 methods. Our write patterns are sufficiently simple that we don't really 

163 need the full flexibility of SQL insert/update/delete syntax, and we need 

164 non-standard (but common) functionality in these operations sufficiently 

165 often that it seems worthwhile to provide our own generic API. 

166 

167 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

168 their SQLAlchemy representation) to be run, as we expect these to require 

169 significantly more sophistication while still being limited to standard 

170 SQL. 

171 

172 `Database` itself has several underscore-prefixed attributes: 

173 

174 - ``_cs``: SQLAlchemy objects representing the connection and transaction 

175 state. 

176 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

177 the tables and other schema entities. 

178 

179 These are considered protected (derived classes may access them, but other 

180 code should not), and read-only, aside from executing SQL via 

181 ``_connection``. 

182 """ 

183 

184 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection, 

185 namespace: Optional[str] = None): 

186 self.origin = origin 

187 self.namespace = namespace 

188 self._connection = connection 

189 self._metadata = None 

190 

191 @classmethod 

192 def makeDefaultUri(cls, root: str) -> Optional[str]: 

193 """Create a default connection URI appropriate for the given root 

194 directory, or `None` if there can be no such default. 

195 """ 

196 return None 

197 

198 @classmethod 

199 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

200 writeable: bool = True) -> Database: 

201 """Construct a database from a SQLAlchemy URI. 

202 

203 Parameters 

204 ---------- 

205 uri : `str` 

206 A SQLAlchemy URI connection string. 

207 origin : `int` 

208 An integer ID that should be used as the default for any datasets, 

209 quanta, or other entities that use a (autoincrement, origin) 

210 compound primary key. 

211 namespace : `str`, optional 

212 A database namespace (i.e. schema) the new instance should be 

213 associated with. If `None` (default), the namespace (if any) is 

214 inferred from the URI. 

215 writeable : `bool`, optional 

216 If `True`, allow write operations on the database, including 

217 ``CREATE TABLE``. 

218 

219 Returns 

220 ------- 

221 db : `Database` 

222 A new `Database` instance. 

223 """ 

224 return cls.fromConnection(cls.connect(uri, writeable=writeable), 

225 origin=origin, 

226 namespace=namespace, 

227 writeable=writeable) 

228 

229 @classmethod 

230 @abstractmethod 

231 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection: 

232 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI. 

233 

234 Parameters 

235 ---------- 

236 uri : `str` 

237 A SQLAlchemy URI connection string. 

238 origin : `int` 

239 An integer ID that should be used as the default for any datasets, 

240 quanta, or other entities that use a (autoincrement, origin) 

241 compound primary key. 

242 writeable : `bool`, optional 

243 If `True`, allow write operations on the database, including 

244 ``CREATE TABLE``. 

245 

246 Returns 

247 ------- 

248 connection : `sqlalchemy.engine.Connection` 

249 A database connection. 

250 

251 Notes 

252 ----- 

253 Subclasses that support other ways to connect to a database are 

254 encouraged to add optional arguments to their implementation of this 

255 method, as long as they maintain compatibility with the base class 

256 call signature. 

257 """ 

258 raise NotImplementedError() 

259 

260 @classmethod 

261 @abstractmethod 

262 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int, 

263 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

264 """Create a new `Database` from an existing 

265 `sqlalchemy.engine.Connection`. 

266 

267 Parameters 

268 ---------- 

269 connection : `sqllachemy.engine.Connection` 

270 The connection for the the database. May be shared between 

271 `Database` instances. 

272 origin : `int` 

273 An integer ID that should be used as the default for any datasets, 

274 quanta, or other entities that use a (autoincrement, origin) 

275 compound primary key. 

276 namespace : `str`, optional 

277 A different database namespace (i.e. schema) the new instance 

278 should be associated with. If `None` (default), the namespace 

279 (if any) is inferred from the connection. 

280 writeable : `bool`, optional 

281 If `True`, allow write operations on the database, including 

282 ``CREATE TABLE``. 

283 

284 Returns 

285 ------- 

286 db : `Database` 

287 A new `Database` instance. 

288 

289 Notes 

290 ----- 

291 This method allows different `Database` instances to share the same 

292 connection, which is desirable when they represent different namespaces 

293 can be queried together. This also ties their transaction state, 

294 however; starting a transaction in any database automatically starts 

295 on in all other databases. 

296 """ 

297 raise NotImplementedError() 

298 

299 @contextmanager 

300 def transaction(self, *, interrupting: bool = False) -> None: 

301 """Return a context manager that represents a transaction. 

302 

303 Parameters 

304 ---------- 

305 interrupting : `bool` 

306 If `True`, this transaction block needs to be able to interrupt 

307 any existing one in order to yield correct behavior. 

308 """ 

309 assert not (interrupting and self._connection.in_transaction()), ( 

310 "Logic error in transaction nesting: an operation that would " 

311 "interrupt the active transaction context has been requested." 

312 ) 

313 if self._connection.in_transaction(): 

314 trans = self._connection.begin_nested() 

315 else: 

316 # Use a regular (non-savepoint) transaction only for the outermost 

317 # context. 

318 trans = self._connection.begin() 

319 try: 

320 yield 

321 trans.commit() 

322 except BaseException: 

323 trans.rollback() 

324 raise 

325 

326 @contextmanager 

327 def declareStaticTables(self, *, create: bool) -> StaticTablesContext: 

328 """Return a context manager in which the database's static DDL schema 

329 can be declared. 

330 

331 Parameters 

332 ---------- 

333 create : `bool` 

334 If `True`, attempt to create all tables at the end of the context. 

335 If `False`, they will be assumed to already exist. 

336 

337 Returns 

338 ------- 

339 schema : `StaticTablesContext` 

340 A helper object that is used to add new tables. 

341 

342 Raises 

343 ------ 

344 ReadOnlyDatabaseError 

345 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

346 and one or more declared tables do not already exist. 

347 

348 Examples 

349 -------- 

350 Given a `Database` instance ``db``:: 

351 

352 with db.declareStaticTables(create=True) as schema: 

353 schema.addTable("table1", TableSpec(...)) 

354 schema.addTable("table2", TableSpec(...)) 

355 

356 Notes 

357 ----- 

358 A database's static DDL schema must be declared before any dynamic 

359 tables are managed via calls to `ensureTableExists` or 

360 `getExistingTable`. The order in which static schema tables are added 

361 inside the context block is unimportant; they will automatically be 

362 sorted and added in an order consistent with their foreign key 

363 relationships. 

364 """ 

365 if create and not self.isWriteable(): 

366 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

367 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

368 try: 

369 context = StaticTablesContext(self) 

370 yield context 

371 for table, foreignKey in context._foreignKeys: 

372 table.append_constraint(foreignKey) 

373 if create: 

374 if self.namespace is not None: 

375 if self.namespace not in context._inspector.get_schema_names(): 

376 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

377 # In our tables we have columns that make use of sqlalchemy 

378 # Sequence objects. There is currently a bug in sqlalchmey that 

379 # causes a deprecation warning to be thrown on a property of 

380 # the Sequence object when the repr for the sequence is 

381 # created. Here a filter is used to catch these deprecation 

382 # warnings when tables are created. 

383 with warnings.catch_warnings(): 

384 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

385 self._metadata.create_all(self._connection) 

386 except BaseException: 

387 self._metadata.drop_all(self._connection) 

388 self._metadata = None 

389 raise 

390 

391 @abstractmethod 

392 def isWriteable(self) -> bool: 

393 """Return `True` if this database can be modified by this client. 

394 """ 

395 raise NotImplementedError() 

396 

397 @abstractmethod 

398 def __str__(self) -> str: 

399 """Return a human-readable identifier for this `Database`, including 

400 any namespace or schema that identifies its names within a `Registry`. 

401 """ 

402 raise NotImplementedError() 

403 

404 def shrinkDatabaseEntityName(self, original: str) -> str: 

405 """Return a version of the given name that fits within this database 

406 engine's length limits for table, constraint, indexes, and sequence 

407 names. 

408 

409 Implementations should not assume that simple truncation is safe, 

410 because multiple long names often begin with the same prefix. 

411 

412 The default implementation simply returns the given name. 

413 

414 Parameters 

415 ---------- 

416 original : `str` 

417 The original name. 

418 

419 Returns 

420 ------- 

421 shrunk : `str` 

422 The new, possibly shortened name. 

423 """ 

424 return original 

425 

426 def expandDatabaseEntityName(self, shrunk: str) -> str: 

427 """Retrieve the original name for a database entity that was too long 

428 to fit within the database engine's limits. 

429 

430 Parameters 

431 ---------- 

432 original : `str` 

433 The original name. 

434 

435 Returns 

436 ------- 

437 shrunk : `str` 

438 The new, possibly shortened name. 

439 """ 

440 return shrunk 

441 

442 def _mangleTableName(self, name: str) -> str: 

443 """Map a logical, user-visible table name to the true table name used 

444 in the database. 

445 

446 The default implementation returns the given name unchanged. 

447 

448 Parameters 

449 ---------- 

450 name : `str` 

451 Input table name. Should not include a namespace (i.e. schema) 

452 prefix. 

453 

454 Returns 

455 ------- 

456 mangled : `str` 

457 Mangled version of the table name (still with no namespace prefix). 

458 

459 Notes 

460 ----- 

461 Reimplementations of this method must be idempotent - mangling an 

462 already-mangled name must have no effect. 

463 """ 

464 return name 

465 

466 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

467 **kwds) -> sqlalchemy.schema.Column: 

468 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

469 

470 Parameters 

471 ---------- 

472 table : `str` 

473 Name of the table this column is being added to. 

474 spec : `FieldSpec` 

475 Specification for the field to be added. 

476 metadata : `sqlalchemy.MetaData` 

477 SQLAlchemy representation of the DDL schema this field's table is 

478 being added to. 

479 **kwds 

480 Additional keyword arguments to forward to the 

481 `sqlalchemy.schema.Column` constructor. This is provided to make 

482 it easier for derived classes to delegate to ``super()`` while 

483 making only minor changes. 

484 

485 Returns 

486 ------- 

487 column : `sqlalchemy.schema.Column` 

488 SQLAlchemy representation of the field. 

489 """ 

490 args = [spec.name, spec.getSizedColumnType()] 

491 if spec.autoincrement: 

492 # Generate a sequence to use for auto incrementing for databases 

493 # that do not support it natively. This will be ignored by 

494 # sqlalchemy for databases that do support it. 

495 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

496 metadata=metadata)) 

497 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

498 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

499 comment=spec.doc, **kwds) 

500 

501 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

502 **kwds) -> sqlalchemy.schema.ForeignKeyConstraint: 

503 """Convert a `ForeignKeySpec` to a 

504 `sqlalchemy.schema.ForeignKeyConstraint`. 

505 

506 Parameters 

507 ---------- 

508 table : `str` 

509 Name of the table this foreign key is being added to. 

510 spec : `ForeignKeySpec` 

511 Specification for the foreign key to be added. 

512 metadata : `sqlalchemy.MetaData` 

513 SQLAlchemy representation of the DDL schema this constraint is 

514 being added to. 

515 **kwds 

516 Additional keyword arguments to forward to the 

517 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

518 provided to make it easier for derived classes to delegate to 

519 ``super()`` while making only minor changes. 

520 

521 Returns 

522 ------- 

523 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

524 SQLAlchemy representation of the constraint. 

525 """ 

526 name = self.shrinkDatabaseEntityName( 

527 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

528 + list(spec.target) + list(spec.source)) 

529 ) 

530 return sqlalchemy.schema.ForeignKeyConstraint( 

531 spec.source, 

532 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

533 name=name, 

534 ondelete=spec.onDelete 

535 ) 

536 

537 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

538 **kwds) -> sqlalchemy.schema.Table: 

539 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

540 

541 Parameters 

542 ---------- 

543 spec : `TableSpec` 

544 Specification for the foreign key to be added. 

545 metadata : `sqlalchemy.MetaData` 

546 SQLAlchemy representation of the DDL schema this table is being 

547 added to. 

548 **kwds 

549 Additional keyword arguments to forward to the 

550 `sqlalchemy.schema.Table` constructor. This is provided to make it 

551 easier for derived classes to delegate to ``super()`` while making 

552 only minor changes. 

553 

554 Returns 

555 ------- 

556 table : `sqlalchemy.schema.Table` 

557 SQLAlchemy representation of the table. 

558 

559 Notes 

560 ----- 

561 This method does not handle ``spec.foreignKeys`` at all, in order to 

562 avoid circular dependencies. These are added by higher-level logic in 

563 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

564 """ 

565 name = self._mangleTableName(name) 

566 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

567 args.extend( 

568 sqlalchemy.schema.UniqueConstraint( 

569 *columns, 

570 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

571 ) 

572 for columns in spec.unique if columns not in spec.indexes 

573 ) 

574 args.extend( 

575 sqlalchemy.schema.Index( 

576 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

577 *columns, 

578 unique=(columns in spec.unique) 

579 ) 

580 for columns in spec.indexes 

581 ) 

582 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

583 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

584 

585 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause: 

586 """Ensure that a table with the given name and specification exists, 

587 creating it if necessary. 

588 

589 Parameters 

590 ---------- 

591 name : `str` 

592 Name of the table (not including namespace qualifiers). 

593 spec : `TableSpec` 

594 Specification for the table. This will be used when creating the 

595 table, and *may* be used when obtaining an existing table to check 

596 for consistency, but no such check is guaranteed. 

597 

598 Returns 

599 ------- 

600 table : `sqlalchemy.schema.Table` 

601 SQLAlchemy representation of the table. 

602 

603 Raises 

604 ------ 

605 ReadOnlyDatabaseError 

606 Raised if `isWriteable` returns `False`, and the table does not 

607 already exist. 

608 DatabaseConflictError 

609 Raised if the table exists but ``spec`` is inconsistent with its 

610 definition. 

611 

612 Notes 

613 ----- 

614 This method may not be called within transactions. It may be called on 

615 read-only databases if and only if the table does in fact already 

616 exist. 

617 

618 Subclasses may override this method, but usually should not need to. 

619 """ 

620 assert not self._connection.in_transaction(), "Table creation interrupts transactions." 

621 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

622 table = self.getExistingTable(name, spec) 

623 if table is not None: 

624 return table 

625 if not self.isWriteable(): 

626 raise ReadOnlyDatabaseError( 

627 f"Table {name} does not exist, and cannot be created " 

628 f"because database {self} is read-only." 

629 ) 

630 table = self._convertTableSpec(name, spec, self._metadata) 

631 for foreignKeySpec in spec.foreignKeys: 

632 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

633 table.create(self._connection) 

634 return table 

635 

636 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

637 """Obtain an existing table with the given name and specification. 

638 

639 Parameters 

640 ---------- 

641 name : `str` 

642 Name of the table (not including namespace qualifiers). 

643 spec : `TableSpec` 

644 Specification for the table. This will be used when creating the 

645 SQLAlchemy representation of the table, and it is used to 

646 check that the actual table in the database is consistent. 

647 

648 Returns 

649 ------- 

650 table : `sqlalchemy.schema.Table` or `None` 

651 SQLAlchemy representation of the table, or `None` if it does not 

652 exist. 

653 

654 Raises 

655 ------ 

656 DatabaseConflictError 

657 Raised if the table exists but ``spec`` is inconsistent with its 

658 definition. 

659 

660 Notes 

661 ----- 

662 This method can be called within transactions and never modifies the 

663 database. 

664 

665 Subclasses may override this method, but usually should not need to. 

666 """ 

667 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

668 name = self._mangleTableName(name) 

669 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

670 if table is not None: 

671 if spec.fields.names != set(table.columns.keys()): 

672 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

673 f"specification has columns {list(spec.fields.names)}, while " 

674 f"the previous definition has {list(table.columns.keys())}.") 

675 else: 

676 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

677 if name in inspector.get_table_names(schema=self.namespace): 

678 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

679 table = self._convertTableSpec(name, spec, self._metadata) 

680 for foreignKeySpec in spec.foreignKeys: 

681 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

682 return table 

683 return table 

684 

685 def sync(self, table: sqlalchemy.schema.Table, *, 

686 keys: Dict[str, Any], 

687 compared: Optional[Dict[str, Any]] = None, 

688 extra: Optional[Dict[str, Any]] = None, 

689 returning: Optional[Sequence[str]] = None, 

690 ) -> Tuple[Optional[Dict[str, Any], bool]]: 

691 """Insert into a table as necessary to ensure database contains 

692 values equivalent to the given ones. 

693 

694 Parameters 

695 ---------- 

696 table : `sqlalchemy.schema.Table` 

697 Table to be queried and possibly inserted into. 

698 keys : `dict` 

699 Column name-value pairs used to search for an existing row; must 

700 be a combination that can be used to select a single row if one 

701 exists. If such a row does not exist, these values are used in 

702 the insert. 

703 compared : `dict`, optional 

704 Column name-value pairs that are compared to those in any existing 

705 row. If such a row does not exist, these rows are used in the 

706 insert. 

707 extra : `dict`, optional 

708 Column name-value pairs that are ignored if a matching row exists, 

709 but used in an insert if one is necessary. 

710 returning : `~collections.abc.Sequence` of `str`, optional 

711 The names of columns whose values should be returned. 

712 

713 Returns 

714 ------- 

715 row : `dict`, optional 

716 The value of the fields indicated by ``returning``, or `None` if 

717 ``returning`` is `None`. 

718 inserted : `bool` 

719 If `True`, a new row was inserted. 

720 

721 Raises 

722 ------ 

723 DatabaseConflictError 

724 Raised if the values in ``compared`` do not match the values in the 

725 database. 

726 ReadOnlyDatabaseError 

727 Raised if `isWriteable` returns `False`, and no matching record 

728 already exists. 

729 

730 Notes 

731 ----- 

732 This method may not be called within transactions. It may be called on 

733 read-only databases if and only if the matching row does in fact 

734 already exist. 

735 """ 

736 

737 def check(): 

738 """Query for a row that matches the ``key`` argument, and compare 

739 to what was given by the caller. 

740 

741 Returns 

742 ------- 

743 n : `int` 

744 Number of matching rows. ``n != 1`` is always an error, but 

745 it's a different kind of error depending on where `check` is 

746 being called. 

747 bad : `list` of `str`, or `None` 

748 The subset of the keys of ``compared`` for which the existing 

749 values did not match the given one. Once again, ``not bad`` 

750 is always an error, but a different kind on context. `None` 

751 if ``n != 1`` 

752 result : `list` or `None` 

753 Results in the database that correspond to the columns given 

754 in ``returning``, or `None` if ``returning is None``. 

755 """ 

756 toSelect = set() 

757 if compared is not None: 

758 toSelect.update(compared.keys()) 

759 if returning is not None: 

760 toSelect.update(returning) 

761 if not toSelect: 

762 # Need to select some column, even if we just want to see 

763 # how many rows we get back. 

764 toSelect.add(next(iter(keys.keys()))) 

765 selectSql = sqlalchemy.sql.select( 

766 [table.columns[k].label(k) for k in toSelect] 

767 ).select_from(table).where( 

768 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

769 ) 

770 fetched = list(self._connection.execute(selectSql).fetchall()) 

771 if len(fetched) != 1: 

772 return len(fetched), None, None 

773 existing = fetched[0] 

774 if compared is not None: 

775 inconsistencies = [k for k, v in compared.items() if existing[k] != v] 

776 else: 

777 inconsistencies = [] 

778 if returning is not None: 

779 toReturn = [existing[k] for k in returning] 

780 else: 

781 toReturn = None 

782 return 1, inconsistencies, toReturn 

783 

784 if self.isWriteable(): 

785 # Database is writeable. Try an insert first, but allow it to fail 

786 # (in only specific ways). 

787 row = keys.copy() 

788 if compared is not None: 

789 row.update(compared) 

790 if extra is not None: 

791 row.update(extra) 

792 insertSql = table.insert().values(row) 

793 try: 

794 with self.transaction(interrupting=True): 

795 self._connection.execute(insertSql) 

796 # Need to perform check() for this branch inside the 

797 # transaction, so we roll back an insert that didn't do 

798 # what we expected. That limits the extent to which we 

799 # can reduce duplication between this block and the other 

800 # ones that perform similar logic. 

801 n, bad, result = check() 

802 if n < 1: 

803 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.") 

804 elif n > 2: 

805 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

806 f"unique constraint for table {table.name}.") 

807 elif bad: 

808 raise RuntimeError("Conflict in sync after successful insert; this should only be " 

809 "possible if the same table is being updated by a concurrent " 

810 "process that isn't using sync.") 

811 # No exceptions, so it looks like we inserted the requested row 

812 # successfully. 

813 inserted = True 

814 except sqlalchemy.exc.IntegrityError as err: 

815 # Most likely cause is that an equivalent row already exists, 

816 # but it could also be some other constraint. Query for the 

817 # row we think we matched to resolve that question. 

818 n, bad, result = check() 

819 if n < 1: 

820 # There was no matched row; insertion failed for some 

821 # completely different reason. Just re-raise the original 

822 # IntegrityError. 

823 raise 

824 elif n > 2: 

825 # There were multiple matched rows, which means we 

826 # conflicted *and* the arguments were bad to begin with. 

827 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

828 f"unique constraint for table {table.name}.") from err 

829 elif bad: 

830 # No logic bug, but data conflicted on the keys given. 

831 raise DatabaseConflictError(f"Conflict in sync for table " 

832 f"{table.name} on column(s) {bad}.") from err 

833 # The desired row is already present and consistent with what 

834 # we tried to insert. 

835 inserted = False 

836 else: 

837 assert not self._connection.in_transaction(), ( 

838 "Calling sync within a transaction block is an error even " 

839 "on a read-only database." 

840 ) 

841 # Database is not writeable; just see if the row exists. 

842 n, bad, result = check() 

843 if n < 1: 

844 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

845 elif n > 1: 

846 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

847 elif bad: 

848 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.") 

849 inserted = False 

850 return {k: v for k, v in zip(returning, result)} if returning is not None else None, inserted 

851 

852 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

853 ) -> Optional[List[int]]: 

854 """Insert one or more rows into a table, optionally returning 

855 autoincrement primary key values. 

856 

857 Parameters 

858 ---------- 

859 table : `sqlalchemy.schema.Table` 

860 Table rows should be inserted into. 

861 returnIds: `bool` 

862 If `True` (`False` is default), return the values of the table's 

863 autoincrement primary key field (which much exist). 

864 *rows 

865 Positional arguments are the rows to be inserted, as dictionaries 

866 mapping column name to value. The keys in all dictionaries must 

867 be the same. 

868 

869 Returns 

870 ------- 

871 ids : `None`, or `list` of `int` 

872 If ``returnIds`` is `True`, a `list` containing the inserted 

873 values for the table's autoincrement primary key. 

874 

875 Raises 

876 ------ 

877 ReadOnlyDatabaseError 

878 Raised if `isWriteable` returns `False` when this method is called. 

879 

880 Notes 

881 ----- 

882 The default implementation uses bulk insert syntax when ``returnIds`` 

883 is `False`, and a loop over single-row insert operations when it is 

884 `True`. 

885 

886 Derived classes should reimplement when they can provide a more 

887 efficient implementation (especially for the latter case). 

888 

889 May be used inside transaction contexts, so implementations may not 

890 perform operations that interrupt transactions. 

891 """ 

892 if not self.isWriteable(): 

893 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.") 

894 if not returnIds: 

895 self._connection.execute(table.insert(), *rows) 

896 else: 

897 sql = table.insert() 

898 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

899 

900 @abstractmethod 

901 def replace(self, table: sqlalchemy.schema.Table, *rows: dict): 

902 """Insert one or more rows into a table, replacing any existing rows 

903 for which insertion of a new row would violate the primary key 

904 constraint. 

905 

906 Parameters 

907 ---------- 

908 table : `sqlalchemy.schema.Table` 

909 Table rows should be inserted into. 

910 *rows 

911 Positional arguments are the rows to be inserted, as dictionaries 

912 mapping column name to value. The keys in all dictionaries must 

913 be the same. 

914 

915 Raises 

916 ------ 

917 ReadOnlyDatabaseError 

918 Raised if `isWriteable` returns `False` when this method is called. 

919 

920 Notes 

921 ----- 

922 May be used inside transaction contexts, so implementations may not 

923 perform operations that interrupt transactions. 

924 

925 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

926 exception when a constraint other than the primary key would be 

927 violated. 

928 

929 Implementations are not required to support `replace` on tables 

930 with autoincrement keys. 

931 """ 

932 raise NotImplementedError() 

933 

934 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

935 """Delete one or more rows from a table. 

936 

937 Parameters 

938 ---------- 

939 table : `sqlalchemy.schema.Table` 

940 Table that rows should be deleted from. 

941 columns: `~collections.abc.Iterable` of `str` 

942 The names of columns that will be used to constrain the rows to 

943 be deleted; these will be combined via ``AND`` to form the 

944 ``WHERE`` clause of the delete query. 

945 *rows 

946 Positional arguments are the keys of rows to be deleted, as 

947 dictionaries mapping column name to value. The keys in all 

948 dictionaries must exactly the names in ``columns``. 

949 

950 Returns 

951 ------- 

952 count : `int` 

953 Number of rows deleted. 

954 

955 Raises 

956 ------ 

957 ReadOnlyDatabaseError 

958 Raised if `isWriteable` returns `False` when this method is called. 

959 

960 Notes 

961 ----- 

962 May be used inside transaction contexts, so implementations may not 

963 perform operations that interrupt transactions. 

964 

965 The default implementation should be sufficient for most derived 

966 classes. 

967 """ 

968 if not self.isWriteable(): 

969 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.") 

970 sql = table.delete() 

971 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

972 if whereTerms: 

973 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

974 return self._connection.execute(sql, *rows).rowcount 

975 

976 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

977 """Update one or more rows in a table. 

978 

979 Parameters 

980 ---------- 

981 table : `sqlalchemy.schema.Table` 

982 Table containing the rows to be updated. 

983 where : `dict` [`str`, `str`] 

984 A mapping from the names of columns that will be used to search for 

985 existing rows to the keys that will hold these values in the 

986 ``rows`` dictionaries. Note that these may not be the same due to 

987 SQLAlchemy limitations. 

988 *rows 

989 Positional arguments are the rows to be updated. The keys in all 

990 dictionaries must be the same, and may correspond to either a 

991 value in the ``where`` dictionary or the name of a column to be 

992 updated. 

993 

994 Returns 

995 ------- 

996 count : `int` 

997 Number of rows matched (regardless of whether the update actually 

998 modified them). 

999 

1000 Raises 

1001 ------ 

1002 ReadOnlyDatabaseError 

1003 Raised if `isWriteable` returns `False` when this method is called. 

1004 

1005 Notes 

1006 ----- 

1007 May be used inside transaction contexts, so implementations may not 

1008 perform operations that interrupt transactions. 

1009 

1010 The default implementation should be sufficient for most derived 

1011 classes. 

1012 """ 

1013 if not self.isWriteable(): 

1014 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.") 

1015 sql = table.update().where( 

1016 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1017 ) 

1018 return self._connection.execute(sql, *rows).rowcount 

1019 

1020 def query(self, sql: sqlalchemy.sql.FromClause, *args, **kwds) -> sqlalchemy.engine.ResultProxy: 

1021 """Run a SELECT query against the database. 

1022 

1023 Parameters 

1024 ---------- 

1025 sql : `sqlalchemy.sql.FromClause` 

1026 A SQLAlchemy representation of a ``SELECT`` query. 

1027 *args 

1028 Additional positional arguments are forwarded to 

1029 `sqlalchemy.engine.Connection.execute`. 

1030 **kwds 

1031 Additional keyword arguments are forwarded to 

1032 `sqlalchemy.engine.Connection.execute`. 

1033 

1034 Returns 

1035 ------- 

1036 result : `sqlalchemy.engine.ResultProxy` 

1037 Query results. 

1038 

1039 Notes 

1040 ----- 

1041 The default implementation should be sufficient for most derived 

1042 classes. 

1043 """ 

1044 # TODO: should we guard against non-SELECT queries here? 

1045 return self._connection.execute(sql, *args, **kwds) 

1046 

1047 origin: int 

1048 """An integer ID that should be used as the default for any datasets, 

1049 quanta, or other entities that use a (autoincrement, origin) compound 

1050 primary key (`int`). 

1051 """ 

1052 

1053 namespace: Optional[str] 

1054 """The schema or namespace this database instance is associated with 

1055 (`str` or `None`). 

1056 """