Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "StaticTablesContext", 

28] 

29 

30from abc import ABC, abstractmethod 

31from contextlib import contextmanager 

32from typing import ( 

33 Any, 

34 Dict, 

35 Iterable, 

36 List, 

37 Optional, 

38 Sequence, 

39 Tuple, 

40) 

41import warnings 

42 

43import sqlalchemy 

44 

45from ...core import ddl 

46 

47 

48def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: Dict[str, Any]): 

49 """Test that the definition of a table in a `ddl.TableSpec` and from 

50 database introspection are consistent. 

51 

52 Parameters 

53 ---------- 

54 name : `str` 

55 Name of the table (only used in error messages). 

56 spec : `ddl.TableSpec` 

57 Specification of the table. 

58 inspection : `dict` 

59 Dictionary returned by 

60 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

61 

62 Raises 

63 ------ 

64 DatabaseConflictError 

65 Raised if the definitions are inconsistent. 

66 """ 

67 columnNames = [c["name"] for c in inspection] 

68 if spec.fields.names != set(columnNames): 

69 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

70 f"specification has columns {list(spec.fields.names)}, while the " 

71 f"table in the database has {columnNames}.") 

72 

73 

74class ReadOnlyDatabaseError(RuntimeError): 

75 """Exception raised when a write operation is called on a read-only 

76 `Database`. 

77 """ 

78 

79 

80class DatabaseConflictError(RuntimeError): 

81 """Exception raised when database content (row values or schema entities) 

82 are inconsistent with what this client expects. 

83 """ 

84 

85 

86class StaticTablesContext: 

87 """Helper class used to declare the static schema for a registry layer 

88 in a database. 

89 

90 An instance of this class is returned by `Database.declareStaticTables`, 

91 which should be the only way it should be constructed. 

92 """ 

93 

94 def __init__(self, db: Database): 

95 self._db = db 

96 self._foreignKeys = [] 

97 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

98 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

99 

100 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

101 """Add a new table to the schema, returning its sqlalchemy 

102 representation. 

103 

104 The new table may not actually be created until the end of the 

105 context created by `Database.declareStaticTables`, allowing tables 

106 to be declared in any order even in the presence of foreign key 

107 relationships. 

108 """ 

109 if name in self._tableNames: 

110 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

111 schema=self._db.namespace)) 

112 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

113 for foreignKeySpec in spec.foreignKeys: 

114 self._foreignKeys.append( 

115 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

116 ) 

117 return table 

118 

119 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

120 """Add a named tuple of tables to the schema, returning their 

121 SQLAlchemy representations in a named tuple of the same type. 

122 

123 The new tables may not actually be created until the end of the 

124 context created by `Database.declareStaticTables`, allowing tables 

125 to be declared in any order even in the presence of foreign key 

126 relationships. 

127 

128 Notes 

129 ----- 

130 ``specs`` *must* be an instance of a type created by 

131 `collections.namedtuple`, not just regular tuple, and the returned 

132 object is guaranteed to be the same. Because `~collections.namedtuple` 

133 is just a factory for `type` objects, not an actual type itself, 

134 we cannot represent this with type annotations. 

135 """ 

136 return specs._make(self.addTable(name, spec) for name, spec in zip(specs._fields, specs)) 

137 

138 

139class Database(ABC): 

140 """An abstract interface that represents a particular database engine's 

141 representation of a single schema/namespace/database. 

142 

143 Parameters 

144 ---------- 

145 origin : `int` 

146 An integer ID that should be used as the default for any datasets, 

147 quanta, or other entities that use a (autoincrement, origin) compound 

148 primary key. 

149 connection : `sqlalchemy.engine.Connection` 

150 The SQLAlchemy connection this `Database` wraps. 

151 namespace : `str`, optional 

152 Name of the schema or namespace this instance is associated with. 

153 This is passed as the ``schema`` argument when constructing a 

154 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

155 avoid confusion between "schema means namespace" and "schema means 

156 table definitions". 

157 

158 Notes 

159 ----- 

160 `Database` requires all write operations to go through its special named 

161 methods. Our write patterns are sufficiently simple that we don't really 

162 need the full flexibility of SQL insert/update/delete syntax, and we need 

163 non-standard (but common) functionality in these operations sufficiently 

164 often that it seems worthwhile to provide our own generic API. 

165 

166 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

167 their SQLAlchemy representation) to be run, as we expect these to require 

168 significantly more sophistication while still being limited to standard 

169 SQL. 

170 

171 `Database` itself has several underscore-prefixed attributes: 

172 

173 - ``_cs``: SQLAlchemy objects representing the connection and transaction 

174 state. 

175 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

176 the tables and other schema entities. 

177 

178 These are considered protected (derived classes may access them, but other 

179 code should not), and read-only, aside from executing SQL via 

180 ``_connection``. 

181 """ 

182 

183 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection, 

184 namespace: Optional[str] = None): 

185 self.origin = origin 

186 self.namespace = namespace 

187 self._connection = connection 

188 self._metadata = None 

189 

190 @classmethod 

191 def makeDefaultUri(cls, root: str) -> Optional[str]: 

192 """Create a default connection URI appropriate for the given root 

193 directory, or `None` if there can be no such default. 

194 """ 

195 return None 

196 

197 @classmethod 

198 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

199 writeable: bool = True) -> Database: 

200 """Construct a database from a SQLAlchemy URI. 

201 

202 Parameters 

203 ---------- 

204 uri : `str` 

205 A SQLAlchemy URI connection string. 

206 origin : `int` 

207 An integer ID that should be used as the default for any datasets, 

208 quanta, or other entities that use a (autoincrement, origin) 

209 compound primary key. 

210 namespace : `str`, optional 

211 A database namespace (i.e. schema) the new instance should be 

212 associated with. If `None` (default), the namespace (if any) is 

213 inferred from the URI. 

214 writeable : `bool`, optional 

215 If `True`, allow write operations on the database, including 

216 ``CREATE TABLE``. 

217 

218 Returns 

219 ------- 

220 db : `Database` 

221 A new `Database` instance. 

222 """ 

223 return cls.fromConnection(cls.connect(uri, writeable=writeable), 

224 origin=origin, 

225 namespace=namespace, 

226 writeable=writeable) 

227 

228 @classmethod 

229 @abstractmethod 

230 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection: 

231 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI. 

232 

233 Parameters 

234 ---------- 

235 uri : `str` 

236 A SQLAlchemy URI connection string. 

237 origin : `int` 

238 An integer ID that should be used as the default for any datasets, 

239 quanta, or other entities that use a (autoincrement, origin) 

240 compound primary key. 

241 writeable : `bool`, optional 

242 If `True`, allow write operations on the database, including 

243 ``CREATE TABLE``. 

244 

245 Returns 

246 ------- 

247 connection : `sqlalchemy.engine.Connection` 

248 A database connection. 

249 

250 Notes 

251 ----- 

252 Subclasses that support other ways to connect to a database are 

253 encouraged to add optional arguments to their implementation of this 

254 method, as long as they maintain compatibility with the base class 

255 call signature. 

256 """ 

257 raise NotImplementedError() 

258 

259 @classmethod 

260 @abstractmethod 

261 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int, 

262 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

263 """Create a new `Database` from an existing 

264 `sqlalchemy.engine.Connection`. 

265 

266 Parameters 

267 ---------- 

268 connection : `sqllachemy.engine.Connection` 

269 The connection for the the database. May be shared between 

270 `Database` instances. 

271 origin : `int` 

272 An integer ID that should be used as the default for any datasets, 

273 quanta, or other entities that use a (autoincrement, origin) 

274 compound primary key. 

275 namespace : `str`, optional 

276 A different database namespace (i.e. schema) the new instance 

277 should be associated with. If `None` (default), the namespace 

278 (if any) is inferred from the connection. 

279 writeable : `bool`, optional 

280 If `True`, allow write operations on the database, including 

281 ``CREATE TABLE``. 

282 

283 Returns 

284 ------- 

285 db : `Database` 

286 A new `Database` instance. 

287 

288 Notes 

289 ----- 

290 This method allows different `Database` instances to share the same 

291 connection, which is desirable when they represent different namespaces 

292 can be queried together. This also ties their transaction state, 

293 however; starting a transaction in any database automatically starts 

294 on in all other databases. 

295 """ 

296 raise NotImplementedError() 

297 

298 @contextmanager 

299 def transaction(self, *, interrupting: bool = False) -> None: 

300 """Return a context manager that represents a transaction. 

301 

302 Parameters 

303 ---------- 

304 interrupting : `bool` 

305 If `True`, this transaction block needs to be able to interrupt 

306 any existing one in order to yield correct behavior. 

307 """ 

308 assert not (interrupting and self._connection.in_transaction()), ( 

309 "Logic error in transaction nesting: an operation that would " 

310 "interrupt the active transaction context has been requested." 

311 ) 

312 if self._connection.in_transaction(): 

313 trans = self._connection.begin_nested() 

314 else: 

315 # Use a regular (non-savepoint) transaction only for the outermost 

316 # context. 

317 trans = self._connection.begin() 

318 try: 

319 yield 

320 trans.commit() 

321 except BaseException: 

322 trans.rollback() 

323 raise 

324 

325 @contextmanager 

326 def declareStaticTables(self, *, create: bool) -> StaticTablesContext: 

327 """Return a context manager in which the database's static DDL schema 

328 can be declared. 

329 

330 Parameters 

331 ---------- 

332 create : `bool` 

333 If `True`, attempt to create all tables at the end of the context. 

334 If `False`, they will be assumed to already exist. 

335 

336 Returns 

337 ------- 

338 schema : `StaticTablesContext` 

339 A helper object that is used to add new tables. 

340 

341 Raises 

342 ------ 

343 ReadOnlyDatabaseError 

344 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

345 and one or more declared tables do not already exist. 

346 

347 Examples 

348 -------- 

349 Given a `Database` instance ``db``:: 

350 

351 with db.declareStaticTables(create=True) as schema: 

352 schema.addTable("table1", TableSpec(...)) 

353 schema.addTable("table2", TableSpec(...)) 

354 

355 Notes 

356 ----- 

357 A database's static DDL schema must be declared before any dynamic 

358 tables are managed via calls to `ensureTableExists` or 

359 `getExistingTable`. The order in which static schema tables are added 

360 inside the context block is unimportant; they will automatically be 

361 sorted and added in an order consistent with their foreign key 

362 relationships. 

363 """ 

364 if create and not self.isWriteable(): 

365 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

366 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

367 try: 

368 context = StaticTablesContext(self) 

369 yield context 

370 for table, foreignKey in context._foreignKeys: 

371 table.append_constraint(foreignKey) 

372 if create: 

373 if self.namespace is not None: 

374 if self.namespace not in context._inspector.get_schema_names(): 

375 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

376 # In our tables we have columns that make use of sqlalchemy 

377 # Sequence objects. There is currently a bug in sqlalchmey that 

378 # causes a deprecation warning to be thrown on a property of 

379 # the Sequence object when the repr for the sequence is 

380 # created. Here a filter is used to catch these deprecation 

381 # warnings when tables are created. 

382 with warnings.catch_warnings(): 

383 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

384 self._metadata.create_all(self._connection) 

385 except BaseException: 

386 self._metadata.drop_all(self._connection) 

387 self._metadata = None 

388 raise 

389 

390 @abstractmethod 

391 def isWriteable(self) -> bool: 

392 """Return `True` if this database can be modified by this client. 

393 """ 

394 raise NotImplementedError() 

395 

396 @abstractmethod 

397 def __str__(self) -> str: 

398 """Return a human-readable identifier for this `Database`, including 

399 any namespace or schema that identifies its names within a `Registry`. 

400 """ 

401 raise NotImplementedError() 

402 

403 def shrinkDatabaseEntityName(self, original: str) -> str: 

404 """Return a version of the given name that fits within this database 

405 engine's length limits for table, constraint, indexes, and sequence 

406 names. 

407 

408 Implementations should not assume that simple truncation is safe, 

409 because multiple long names often begin with the same prefix. 

410 

411 The default implementation simply returns the given name. 

412 

413 Parameters 

414 ---------- 

415 original : `str` 

416 The original name. 

417 

418 Returns 

419 ------- 

420 shrunk : `str` 

421 The new, possibly shortened name. 

422 """ 

423 return original 

424 

425 def expandDatabaseEntityName(self, shrunk: str) -> str: 

426 """Retrieve the original name for a database entity that was too long 

427 to fit within the database engine's limits. 

428 

429 Parameters 

430 ---------- 

431 original : `str` 

432 The original name. 

433 

434 Returns 

435 ------- 

436 shrunk : `str` 

437 The new, possibly shortened name. 

438 """ 

439 return shrunk 

440 

441 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

442 **kwds) -> sqlalchemy.schema.Column: 

443 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

444 

445 Parameters 

446 ---------- 

447 table : `str` 

448 Name of the table this column is being added to. 

449 spec : `FieldSpec` 

450 Specification for the field to be added. 

451 metadata : `sqlalchemy.MetaData` 

452 SQLAlchemy representation of the DDL schema this field's table is 

453 being added to. 

454 **kwds 

455 Additional keyword arguments to forward to the 

456 `sqlalchemy.schema.Column` constructor. This is provided to make 

457 it easier for derived classes to delegate to ``super()`` while 

458 making only minor changes. 

459 

460 Returns 

461 ------- 

462 column : `sqlalchemy.schema.Column` 

463 SQLAlchemy representation of the field. 

464 """ 

465 args = [spec.name, spec.getSizedColumnType()] 

466 if spec.autoincrement: 

467 # Generate a sequence to use for auto incrementing for databases 

468 # that do not support it natively. This will be ignored by 

469 # sqlalchemy for databases that do support it. 

470 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

471 metadata=metadata)) 

472 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

473 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

474 comment=spec.doc, **kwds) 

475 

476 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

477 **kwds) -> sqlalchemy.schema.ForeignKeyConstraint: 

478 """Convert a `ForeignKeySpec` to a 

479 `sqlalchemy.schema.ForeignKeyConstraint`. 

480 

481 Parameters 

482 ---------- 

483 table : `str` 

484 Name of the table this foreign key is being added to. 

485 spec : `ForeignKeySpec` 

486 Specification for the foreign key to be added. 

487 metadata : `sqlalchemy.MetaData` 

488 SQLAlchemy representation of the DDL schema this constraint is 

489 being added to. 

490 **kwds 

491 Additional keyword arguments to forward to the 

492 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

493 provided to make it easier for derived classes to delegate to 

494 ``super()`` while making only minor changes. 

495 

496 Returns 

497 ------- 

498 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

499 SQLAlchemy representation of the constraint. 

500 """ 

501 name = self.shrinkDatabaseEntityName( 

502 "_".join(["fkey", table, spec.table] + list(spec.target) + list(spec.source)) 

503 ) 

504 return sqlalchemy.schema.ForeignKeyConstraint( 

505 spec.source, 

506 [f"{spec.table}.{col}" for col in spec.target], 

507 name=name, 

508 ondelete=spec.onDelete 

509 ) 

510 

511 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

512 **kwds) -> sqlalchemy.schema.Table: 

513 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

514 

515 Parameters 

516 ---------- 

517 spec : `TableSpec` 

518 Specification for the foreign key to be added. 

519 metadata : `sqlalchemy.MetaData` 

520 SQLAlchemy representation of the DDL schema this table is being 

521 added to. 

522 **kwds 

523 Additional keyword arguments to forward to the 

524 `sqlalchemy.schema.Table` constructor. This is provided to make it 

525 easier for derived classes to delegate to ``super()`` while making 

526 only minor changes. 

527 

528 Returns 

529 ------- 

530 table : `sqlalchemy.schema.Table` 

531 SQLAlchemy representation of the table. 

532 

533 Notes 

534 ----- 

535 This method does not handle ``spec.foreignKeys`` at all, in order to 

536 avoid circular dependencies. These are added by higher-level logic in 

537 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

538 """ 

539 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

540 args.extend( 

541 sqlalchemy.schema.UniqueConstraint( 

542 *columns, 

543 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

544 ) 

545 for columns in spec.unique if columns not in spec.indexes 

546 ) 

547 args.extend( 

548 sqlalchemy.schema.Index( 

549 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

550 *columns, 

551 unique=(columns in spec.unique) 

552 ) 

553 for columns in spec.indexes 

554 ) 

555 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

556 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

557 

558 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause: 

559 """Ensure that a table with the given name and specification exists, 

560 creating it if necessary. 

561 

562 Parameters 

563 ---------- 

564 name : `str` 

565 Name of the table (not including namespace qualifiers). 

566 spec : `TableSpec` 

567 Specification for the table. This will be used when creating the 

568 table, and *may* be used when obtaining an existing table to check 

569 for consistency, but no such check is guaranteed. 

570 

571 Returns 

572 ------- 

573 table : `sqlalchemy.schema.Table` 

574 SQLAlchemy representation of the table. 

575 

576 Raises 

577 ------ 

578 ReadOnlyDatabaseError 

579 Raised if `isWriteable` returns `False`, and the table does not 

580 already exist. 

581 DatabaseConflictError 

582 Raised if the table exists but ``spec`` is inconsistent with its 

583 definition. 

584 

585 Notes 

586 ----- 

587 This method may not be called within transactions. It may be called on 

588 read-only databases if and only if the table does in fact already 

589 exist. 

590 

591 Subclasses may override this method, but usually should not need to. 

592 """ 

593 assert not self._connection.in_transaction(), "Table creation interrupts transactions." 

594 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

595 table = self.getExistingTable(name, spec) 

596 if table is not None: 

597 return table 

598 if not self.isWriteable(): 

599 raise ReadOnlyDatabaseError( 

600 f"Table {name} does not exist, and cannot be created " 

601 f"because database {self} is read-only." 

602 ) 

603 table = self._convertTableSpec(name, spec, self._metadata) 

604 for foreignKeySpec in spec.foreignKeys: 

605 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

606 table.create(self._connection) 

607 return table 

608 

609 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

610 """Obtain an existing table with the given name and specification. 

611 

612 Parameters 

613 ---------- 

614 name : `str` 

615 Name of the table (not including namespace qualifiers). 

616 spec : `TableSpec` 

617 Specification for the table. This will be used when creating the 

618 SQLAlchemy representation of the table, and it is used to 

619 check that the actual table in the database is consistent. 

620 

621 Returns 

622 ------- 

623 table : `sqlalchemy.schema.Table` or `None` 

624 SQLAlchemy representation of the table, or `None` if it does not 

625 exist. 

626 

627 Raises 

628 ------ 

629 DatabaseConflictError 

630 Raised if the table exists but ``spec`` is inconsistent with its 

631 definition. 

632 

633 Notes 

634 ----- 

635 This method can be called within transactions and never modifies the 

636 database. 

637 

638 Subclasses may override this method, but usually should not need to. 

639 """ 

640 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

641 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

642 if table is not None: 

643 if spec.fields.names != set(table.columns.keys()): 

644 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

645 f"specification has columns {list(spec.fields.names)}, while " 

646 f"the previous definition has {list(table.columns.keys())}.") 

647 else: 

648 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

649 if name in inspector.get_table_names(schema=self.namespace): 

650 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

651 table = self._convertTableSpec(name, spec, self._metadata) 

652 for foreignKeySpec in spec.foreignKeys: 

653 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

654 return table 

655 return table 

656 

657 def sync(self, table: sqlalchemy.schema.Table, *, 

658 keys: Dict[str, Any], 

659 compared: Optional[Dict[str, Any]] = None, 

660 extra: Optional[Dict[str, Any]] = None, 

661 returning: Optional[Sequence[str]] = None, 

662 ) -> Tuple[Optional[Dict[str, Any], bool]]: 

663 """Insert into a table as necessary to ensure database contains 

664 values equivalent to the given ones. 

665 

666 Parameters 

667 ---------- 

668 table : `sqlalchemy.schema.Table` 

669 Table to be queried and possibly inserted into. 

670 keys : `dict` 

671 Column name-value pairs used to search for an existing row; must 

672 be a combination that can be used to select a single row if one 

673 exists. If such a row does not exist, these values are used in 

674 the insert. 

675 compared : `dict`, optional 

676 Column name-value pairs that are compared to those in any existing 

677 row. If such a row does not exist, these rows are used in the 

678 insert. 

679 extra : `dict`, optional 

680 Column name-value pairs that are ignored if a matching row exists, 

681 but used in an insert if one is necessary. 

682 returning : `~collections.abc.Sequence` of `str`, optional 

683 The names of columns whose values should be returned. 

684 

685 Returns 

686 ------- 

687 row : `dict`, optional 

688 The value of the fields indicated by ``returning``, or `None` if 

689 ``returning`` is `None`. 

690 inserted : `bool` 

691 If `True`, a new row was inserted. 

692 

693 Raises 

694 ------ 

695 DatabaseConflictErrorError 

696 Raised if the values in ``compared`` do match the values in the 

697 database. 

698 ReadOnlyDatabaseError 

699 Raised if `isWriteable` returns `False`, and no matching record 

700 already exists. 

701 

702 Notes 

703 ----- 

704 This method may not be called within transactions. It may be called on 

705 read-only databases if and only if the matching row does in fact 

706 already exist. 

707 """ 

708 

709 def check(): 

710 """Query for a row that matches the ``key`` argument, and compare 

711 to what was given by the caller. 

712 

713 Returns 

714 ------- 

715 n : `int` 

716 Number of matching rows. ``n != 1`` is always an error, but 

717 it's a different kind of error depending on where `check` is 

718 being called. 

719 bad : `list` of `str`, or `None` 

720 The subset of the keys of ``compared`` for which the existing 

721 values did not match the given one. Once again, ``not bad`` 

722 is always an error, but a different kind on context. `None` 

723 if ``n != 1`` 

724 result : `list` or `None` 

725 Results in the database that correspond to the columns given 

726 in ``returning``, or `None` if ``returning is None``. 

727 """ 

728 toSelect = set() 

729 if compared is not None: 

730 toSelect.update(compared.keys()) 

731 if returning is not None: 

732 toSelect.update(returning) 

733 if not toSelect: 

734 # Need to select some column, even if we just want to see 

735 # how many rows we get back. 

736 toSelect.add(next(iter(keys.keys()))) 

737 selectSql = sqlalchemy.sql.select( 

738 [table.columns[k].label(k) for k in toSelect] 

739 ).select_from(table).where( 

740 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

741 ) 

742 fetched = list(self._connection.execute(selectSql).fetchall()) 

743 if len(fetched) != 1: 

744 return len(fetched), None, None 

745 existing = fetched[0] 

746 if compared is not None: 

747 inconsistencies = [k for k, v in compared.items() if existing[k] != v] 

748 else: 

749 inconsistencies = [] 

750 if returning is not None: 

751 toReturn = [existing[k] for k in returning] 

752 else: 

753 toReturn = None 

754 return 1, inconsistencies, toReturn 

755 

756 if self.isWriteable(): 

757 # Database is writeable. Try an insert first, but allow it to fail 

758 # (in only specific ways). 

759 row = keys.copy() 

760 if compared is not None: 

761 row.update(compared) 

762 if extra is not None: 

763 row.update(extra) 

764 insertSql = table.insert().values(row) 

765 try: 

766 with self.transaction(interrupting=True): 

767 self._connection.execute(insertSql) 

768 # Need to perform check() for this branch inside the 

769 # transaction, so we roll back an insert that didn't do 

770 # what we expected. That limits the extent to which we 

771 # can reduce duplication between this block and the other 

772 # ones that perform similar logic. 

773 n, bad, result = check() 

774 if n < 1: 

775 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.") 

776 elif n > 2: 

777 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

778 f"unique constraint for table {table.name}.") 

779 elif bad: 

780 raise RuntimeError("Conflict in sync after successful insert; this should only be " 

781 "possible if the same table is being updated by a concurrent " 

782 "process that isn't using sync.") 

783 # No exceptions, so it looks like we inserted the requested row 

784 # successfully. 

785 inserted = True 

786 except sqlalchemy.exc.IntegrityError as err: 

787 # Most likely cause is that an equivalent row already exists, 

788 # but it could also be some other constraint. Query for the 

789 # row we think we matched to resolve that question. 

790 n, bad, result = check() 

791 if n < 1: 

792 # There was no matched row; insertion failed for some 

793 # completely different reason. Just re-raise the original 

794 # IntegrityError. 

795 raise 

796 elif n > 2: 

797 # There were multiple matched rows, which means we 

798 # conflicted *and* the arguments were bad to begin with. 

799 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

800 f"unique constraint for table {table.name}.") from err 

801 elif bad: 

802 # No logic bug, but data conflicted on the keys given. 

803 raise DatabaseConflictError(f"Conflict in sync for table " 

804 f"{table.name} on column(s) {bad}.") from err 

805 # The desired row is already present and consistent with what 

806 # we tried to insert. 

807 inserted = False 

808 else: 

809 assert not self._connection.in_transaction(), ( 

810 "Calling sync within a transaction block is an error even " 

811 "on a read-only database." 

812 ) 

813 # Database is not writeable; just see if the row exists. 

814 n, bad, result = check() 

815 if n < 1: 

816 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

817 elif n > 1: 

818 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

819 elif bad: 

820 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.") 

821 inserted = False 

822 return result, inserted 

823 

824 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

825 ) -> Optional[List[int]]: 

826 """Insert one or more rows into a table, optionally returning 

827 autoincrement primary key values. 

828 

829 Parameters 

830 ---------- 

831 table : `sqlalchemy.schema.Table` 

832 Table rows should be inserted into. 

833 returnIds: `bool` 

834 If `True` (`False` is default), return the values of the table's 

835 autoincrement primary key field (which much exist). 

836 *rows 

837 Positional arguments are the rows to be inserted, as dictionaries 

838 mapping column name to value. The keys in all dictionaries must 

839 be the same. 

840 

841 Returns 

842 ------- 

843 ids : `None`, or `list` of `int` 

844 If ``returnIds`` is `True`, a `list` containing the inserted 

845 values for the table's autoincrement primary key. 

846 

847 Raises 

848 ------ 

849 ReadOnlyDatabaseError 

850 Raised if `isWriteable` returns `False` when this method is called. 

851 

852 Notes 

853 ----- 

854 The default implementation uses bulk insert syntax when ``returnIds`` 

855 is `False`, and a loop over single-row insert operations when it is 

856 `True`. 

857 

858 Derived classes should reimplement when they can provide a more 

859 efficient implementation (especially for the latter case). 

860 

861 May be used inside transaction contexts, so implementations may not 

862 perform operations that interrupt transactions. 

863 """ 

864 if not self.isWriteable(): 

865 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.") 

866 if not returnIds: 

867 self._connection.execute(table.insert(), *rows) 

868 else: 

869 sql = table.insert() 

870 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

871 

872 @abstractmethod 

873 def replace(self, table: sqlalchemy.schema.Table, *rows: dict): 

874 """Insert one or more rows into a table, replacing any existing rows 

875 for which insertion of a new row would violate the primary key 

876 constraint. 

877 

878 Parameters 

879 ---------- 

880 table : `sqlalchemy.schema.Table` 

881 Table rows should be inserted into. 

882 *rows 

883 Positional arguments are the rows to be inserted, as dictionaries 

884 mapping column name to value. The keys in all dictionaries must 

885 be the same. 

886 

887 Raises 

888 ------ 

889 ReadOnlyDatabaseError 

890 Raised if `isWriteable` returns `False` when this method is called. 

891 

892 Notes 

893 ----- 

894 May be used inside transaction contexts, so implementations may not 

895 perform operations that interrupt transactions. 

896 

897 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

898 exception when a constraint other than the primary key would be 

899 violated. 

900 

901 Implementations are not required to support `replace` on tables 

902 with autoincrement keys. 

903 """ 

904 raise NotImplementedError() 

905 

906 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

907 """Delete one or more rows from a table. 

908 

909 Parameters 

910 ---------- 

911 table : `sqlalchemy.schema.Table` 

912 Table that rows should be deleted from. 

913 columns: `~collections.abc.Iterable` of `str` 

914 The names of columns that will be used to constrain the rows to 

915 be deleted; these will be combined via ``AND`` to form the 

916 ``WHERE`` clause of the delete query. 

917 *rows 

918 Positional arguments are the keys of rows to be deleted, as 

919 dictionaries mapping column name to value. The keys in all 

920 dictionaries must exactly the names in ``columns``. 

921 

922 Returns 

923 ------- 

924 count : `int` 

925 Number of rows deleted. 

926 

927 Raises 

928 ------ 

929 ReadOnlyDatabaseError 

930 Raised if `isWriteable` returns `False` when this method is called. 

931 

932 Notes 

933 ----- 

934 May be used inside transaction contexts, so implementations may not 

935 perform operations that interrupt transactions. 

936 

937 The default implementation should be sufficient for most derived 

938 classes. 

939 """ 

940 if not self.isWriteable(): 

941 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.") 

942 sql = table.delete() 

943 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

944 if whereTerms: 

945 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

946 return self._connection.execute(sql, *rows).rowcount 

947 

948 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

949 """Update one or more rows in a table. 

950 

951 Parameters 

952 ---------- 

953 table : `sqlalchemy.schema.Table` 

954 Table containing the rows to be updated. 

955 where : `dict` [`str`, `str`] 

956 A mapping from the names of columns that will be used to search for 

957 existing rows to the keys that will hold these values in the 

958 ``rows`` dictionaries. Note that these may not be the same due to 

959 SQLAlchemy limitations. 

960 *rows 

961 Positional arguments are the rows to be updated. The keys in all 

962 dictionaries must be the same, and may correspond to either a 

963 value in the ``where`` dictionary or the name of a column to be 

964 updated. 

965 

966 Raises 

967 ------ 

968 ReadOnlyDatabaseError 

969 Raised if `isWriteable` returns `False` when this method is called. 

970 

971 Returns 

972 ------- 

973 count : `int` 

974 Number of rows matched (regardless of whether the update actually 

975 modified them). 

976 

977 Notes 

978 ----- 

979 May be used inside transaction contexts, so implementations may not 

980 perform operations that interrupt transactions. 

981 

982 The default implementation should be sufficient for most derived 

983 classes. 

984 """ 

985 if not self.isWriteable(): 

986 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.") 

987 sql = table.update().where( 

988 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

989 ) 

990 return self._connection.execute(sql, *rows).rowcount 

991 

992 def query(self, sql: sqlalchemy.sql.FromClause, *args, **kwds) -> sqlalchemy.engine.ResultProxy: 

993 """Run a SELECT query against the database. 

994 

995 Parameters 

996 ---------- 

997 sql : `sqlalchemy.sql.FromClause` 

998 A SQLAlchemy representation of a ``SELECT`` query. 

999 *args 

1000 Additional positional arguments are forwarded to 

1001 `sqlalchemy.engine.Connection.execute`. 

1002 **kwds 

1003 Additional keyword arguments are forwarded to 

1004 `sqlalchemy.engine.Connection.execute`. 

1005 

1006 Returns 

1007 ------- 

1008 result : `sqlalchemy.engine.ResultProxy` 

1009 Query results. 

1010 

1011 Notes 

1012 ----- 

1013 The default implementation should be sufficient for most derived 

1014 classes. 

1015 """ 

1016 # TODO: should we guard against non-SELECT queries here? 

1017 return self._connection.execute(sql, *args, **kwds) 

1018 

1019 origin: int 

1020 """An integer ID that should be used as the default for any datasets, 

1021 quanta, or other entities that use a (autoincrement, origin) compound 

1022 primary key (`int`). 

1023 """ 

1024 

1025 namespace: Optional[str] 

1026 """The schema or namespace this database instance is associated with 

1027 (`str` or `None`). 

1028 """