Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "SchemaAlreadyDefinedError", 

28 "StaticTablesContext", 

29] 

30 

31from abc import ABC, abstractmethod 

32from contextlib import contextmanager 

33from typing import ( 

34 Any, 

35 Callable, 

36 Dict, 

37 Iterable, 

38 Iterator, 

39 List, 

40 Optional, 

41 Sequence, 

42 Set, 

43 Tuple, 

44 Type, 

45 Union, 

46) 

47import uuid 

48import warnings 

49 

50import astropy.time 

51import sqlalchemy 

52 

53from ...core import SpatialRegionDatabaseRepresentation, TimespanDatabaseRepresentation, ddl, time_utils 

54from .._exceptions import ConflictingDefinitionError 

55 

56_IN_SAVEPOINT_TRANSACTION = "IN_SAVEPOINT_TRANSACTION" 

57 

58 

59def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None: 

60 """Test that the definition of a table in a `ddl.TableSpec` and from 

61 database introspection are consistent. 

62 

63 Parameters 

64 ---------- 

65 name : `str` 

66 Name of the table (only used in error messages). 

67 spec : `ddl.TableSpec` 

68 Specification of the table. 

69 inspection : `dict` 

70 Dictionary returned by 

71 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

72 

73 Raises 

74 ------ 

75 DatabaseConflictError 

76 Raised if the definitions are inconsistent. 

77 """ 

78 columnNames = [c["name"] for c in inspection] 

79 if spec.fields.names != set(columnNames): 

80 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

81 f"specification has columns {list(spec.fields.names)}, while the " 

82 f"table in the database has {columnNames}.") 

83 

84 

85class ReadOnlyDatabaseError(RuntimeError): 

86 """Exception raised when a write operation is called on a read-only 

87 `Database`. 

88 """ 

89 

90 

91class DatabaseConflictError(ConflictingDefinitionError): 

92 """Exception raised when database content (row values or schema entities) 

93 are inconsistent with what this client expects. 

94 """ 

95 

96 

97class SchemaAlreadyDefinedError(RuntimeError): 

98 """Exception raised when trying to initialize database schema when some 

99 tables already exist. 

100 """ 

101 

102 

103class StaticTablesContext: 

104 """Helper class used to declare the static schema for a registry layer 

105 in a database. 

106 

107 An instance of this class is returned by `Database.declareStaticTables`, 

108 which should be the only way it should be constructed. 

109 """ 

110 

111 def __init__(self, db: Database): 

112 self._db = db 

113 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = [] 

114 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

115 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

116 self._initializers: List[Callable[[Database], None]] = [] 

117 

118 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

119 """Add a new table to the schema, returning its sqlalchemy 

120 representation. 

121 

122 The new table may not actually be created until the end of the 

123 context created by `Database.declareStaticTables`, allowing tables 

124 to be declared in any order even in the presence of foreign key 

125 relationships. 

126 """ 

127 name = self._db._mangleTableName(name) 

128 if name in self._tableNames: 

129 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

130 schema=self._db.namespace)) 

131 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

132 for foreignKeySpec in spec.foreignKeys: 

133 self._foreignKeys.append( 

134 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

135 ) 

136 return table 

137 

138 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

139 """Add a named tuple of tables to the schema, returning their 

140 SQLAlchemy representations in a named tuple of the same type. 

141 

142 The new tables may not actually be created until the end of the 

143 context created by `Database.declareStaticTables`, allowing tables 

144 to be declared in any order even in the presence of foreign key 

145 relationships. 

146 

147 Notes 

148 ----- 

149 ``specs`` *must* be an instance of a type created by 

150 `collections.namedtuple`, not just regular tuple, and the returned 

151 object is guaranteed to be the same. Because `~collections.namedtuple` 

152 is just a factory for `type` objects, not an actual type itself, 

153 we cannot represent this with type annotations. 

154 """ 

155 return specs._make(self.addTable(name, spec) # type: ignore 

156 for name, spec in zip(specs._fields, specs)) # type: ignore 

157 

158 def addInitializer(self, initializer: Callable[[Database], None]) -> None: 

159 """Add a method that does one-time initialization of a database. 

160 

161 Initialization can mean anything that changes state of a database 

162 and needs to be done exactly once after database schema was created. 

163 An example for that could be population of schema attributes. 

164 

165 Parameters 

166 ---------- 

167 initializer : callable 

168 Method of a single argument which is a `Database` instance. 

169 """ 

170 self._initializers.append(initializer) 

171 

172 

173class Session: 

174 """Class representing a persistent connection to a database. 

175 

176 Parameters 

177 ---------- 

178 db : `Database` 

179 Database instance. 

180 

181 Notes 

182 ----- 

183 Instances of Session class should not be created by client code; 

184 `Database.session` should be used to create context for a session:: 

185 

186 with db.session() as session: 

187 session.method() 

188 db.method() 

189 

190 In the current implementation sessions can be nested and transactions can 

191 be nested within a session. All nested sessions and transaction share the 

192 same database connection. 

193 

194 Session class represents a limited subset of database API that requires 

195 persistent connection to a database (e.g. temporary tables which have 

196 lifetime of a session). Potentially most of the database API could be 

197 associated with a Session class. 

198 """ 

199 def __init__(self, db: Database): 

200 self._db = db 

201 

202 def makeTemporaryTable(self, spec: ddl.TableSpec, name: Optional[str] = None) -> sqlalchemy.schema.Table: 

203 """Create a temporary table. 

204 

205 Parameters 

206 ---------- 

207 spec : `TableSpec` 

208 Specification for the table. 

209 name : `str`, optional 

210 A unique (within this session/connetion) name for the table. 

211 Subclasses may override to modify the actual name used. If not 

212 provided, a unique name will be generated. 

213 

214 Returns 

215 ------- 

216 table : `sqlalchemy.schema.Table` 

217 SQLAlchemy representation of the table. 

218 

219 Notes 

220 ----- 

221 Temporary tables may be created, dropped, and written to even in 

222 read-only databases - at least according to the Python-level 

223 protections in the `Database` classes. Server permissions may say 

224 otherwise, but in that case they probably need to be modified to 

225 support the full range of expected read-only butler behavior. 

226 

227 Temporary table rows are guaranteed to be dropped when a connection is 

228 closed. `Database` implementations are permitted to allow the table to 

229 remain as long as this is transparent to the user (i.e. "creating" the 

230 temporary table in a new session should not be an error, even if it 

231 does nothing). 

232 

233 It may not be possible to use temporary tables within transactions with 

234 some database engines (or configurations thereof). 

235 """ 

236 if name is None: 

237 name = f"tmp_{uuid.uuid4().hex}" 

238 table = self._db._convertTableSpec(name, spec, self._db._metadata, prefixes=['TEMPORARY'], 

239 schema=sqlalchemy.schema.BLANK_SCHEMA) 

240 if table.key in self._db._tempTables: 

241 if table.key != name: 

242 raise ValueError(f"A temporary table with name {name} (transformed to {table.key} by " 

243 f"Database) already exists.") 

244 for foreignKeySpec in spec.foreignKeys: 

245 table.append_constraint(self._db._convertForeignKeySpec(name, foreignKeySpec, 

246 self._db._metadata)) 

247 table.create(self._db._session_connection) 

248 self._db._tempTables.add(table.key) 

249 return table 

250 

251 def dropTemporaryTable(self, table: sqlalchemy.schema.Table) -> None: 

252 """Drop a temporary table. 

253 

254 Parameters 

255 ---------- 

256 table : `sqlalchemy.schema.Table` 

257 A SQLAlchemy object returned by a previous call to 

258 `makeTemporaryTable`. 

259 """ 

260 if table.key in self._db._tempTables: 

261 table.drop(self._db._session_connection) 

262 self._db._tempTables.remove(table.key) 

263 else: 

264 raise TypeError(f"Table {table.key} was not created by makeTemporaryTable.") 

265 

266 

267class Database(ABC): 

268 """An abstract interface that represents a particular database engine's 

269 representation of a single schema/namespace/database. 

270 

271 Parameters 

272 ---------- 

273 origin : `int` 

274 An integer ID that should be used as the default for any datasets, 

275 quanta, or other entities that use a (autoincrement, origin) compound 

276 primary key. 

277 engine : `sqlalchemy.engine.Engine` 

278 The SQLAlchemy engine for this `Database`. 

279 namespace : `str`, optional 

280 Name of the schema or namespace this instance is associated with. 

281 This is passed as the ``schema`` argument when constructing a 

282 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

283 avoid confusion between "schema means namespace" and "schema means 

284 table definitions". 

285 

286 Notes 

287 ----- 

288 `Database` requires all write operations to go through its special named 

289 methods. Our write patterns are sufficiently simple that we don't really 

290 need the full flexibility of SQL insert/update/delete syntax, and we need 

291 non-standard (but common) functionality in these operations sufficiently 

292 often that it seems worthwhile to provide our own generic API. 

293 

294 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

295 their SQLAlchemy representation) to be run, as we expect these to require 

296 significantly more sophistication while still being limited to standard 

297 SQL. 

298 

299 `Database` itself has several underscore-prefixed attributes: 

300 

301 - ``_engine``: SQLAlchemy object representing its engine. 

302 - ``_connection``: the `sqlalchemy.engine.Connectable` object which can 

303 be either an Engine or Connection if a session is active. 

304 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

305 the tables and other schema entities. 

306 

307 These are considered protected (derived classes may access them, but other 

308 code should not), and read-only, aside from executing SQL via 

309 ``_connection``. 

310 """ 

311 

312 def __init__(self, *, origin: int, engine: sqlalchemy.engine.Engine, 

313 namespace: Optional[str] = None): 

314 self.origin = origin 

315 self.namespace = namespace 

316 self._engine = engine 

317 self._session_connection: Optional[sqlalchemy.engine.Connection] = None 

318 self._metadata: Optional[sqlalchemy.schema.MetaData] = None 

319 self._tempTables: Set[str] = set() 

320 

321 def __repr__(self) -> str: 

322 # Rather than try to reproduce all the parameters used to create 

323 # the object, instead report the more useful information of the 

324 # connection URL. 

325 uri = str(self._engine.url) 

326 if self.namespace: 

327 uri += f"#{self.namespace}" 

328 return f'{type(self).__name__}("{uri}")' 

329 

330 @classmethod 

331 def makeDefaultUri(cls, root: str) -> Optional[str]: 

332 """Create a default connection URI appropriate for the given root 

333 directory, or `None` if there can be no such default. 

334 """ 

335 return None 

336 

337 @classmethod 

338 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

339 writeable: bool = True) -> Database: 

340 """Construct a database from a SQLAlchemy URI. 

341 

342 Parameters 

343 ---------- 

344 uri : `str` 

345 A SQLAlchemy URI connection string. 

346 origin : `int` 

347 An integer ID that should be used as the default for any datasets, 

348 quanta, or other entities that use a (autoincrement, origin) 

349 compound primary key. 

350 namespace : `str`, optional 

351 A database namespace (i.e. schema) the new instance should be 

352 associated with. If `None` (default), the namespace (if any) is 

353 inferred from the URI. 

354 writeable : `bool`, optional 

355 If `True`, allow write operations on the database, including 

356 ``CREATE TABLE``. 

357 

358 Returns 

359 ------- 

360 db : `Database` 

361 A new `Database` instance. 

362 """ 

363 return cls.fromEngine(cls.makeEngine(uri, writeable=writeable), 

364 origin=origin, 

365 namespace=namespace, 

366 writeable=writeable) 

367 

368 @classmethod 

369 @abstractmethod 

370 def makeEngine(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Engine: 

371 """Create a `sqlalchemy.engine.Engine` from a SQLAlchemy URI. 

372 

373 Parameters 

374 ---------- 

375 uri : `str` 

376 A SQLAlchemy URI connection string. 

377 writeable : `bool`, optional 

378 If `True`, allow write operations on the database, including 

379 ``CREATE TABLE``. 

380 

381 Returns 

382 ------- 

383 engine : `sqlalchemy.engine.Engine` 

384 A database engine. 

385 

386 Notes 

387 ----- 

388 Subclasses that support other ways to connect to a database are 

389 encouraged to add optional arguments to their implementation of this 

390 method, as long as they maintain compatibility with the base class 

391 call signature. 

392 """ 

393 raise NotImplementedError() 

394 

395 @classmethod 

396 @abstractmethod 

397 def fromEngine(cls, engine: sqlalchemy.engine.Engine, *, origin: int, 

398 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

399 """Create a new `Database` from an existing `sqlalchemy.engine.Engine`. 

400 

401 Parameters 

402 ---------- 

403 engine : `sqllachemy.engine.Engine` 

404 The engine for the database. May be shared between `Database` 

405 instances. 

406 origin : `int` 

407 An integer ID that should be used as the default for any datasets, 

408 quanta, or other entities that use a (autoincrement, origin) 

409 compound primary key. 

410 namespace : `str`, optional 

411 A different database namespace (i.e. schema) the new instance 

412 should be associated with. If `None` (default), the namespace 

413 (if any) is inferred from the connection. 

414 writeable : `bool`, optional 

415 If `True`, allow write operations on the database, including 

416 ``CREATE TABLE``. 

417 

418 Returns 

419 ------- 

420 db : `Database` 

421 A new `Database` instance. 

422 

423 Notes 

424 ----- 

425 This method allows different `Database` instances to share the same 

426 engine, which is desirable when they represent different namespaces 

427 can be queried together. 

428 """ 

429 raise NotImplementedError() 

430 

431 @contextmanager 

432 def session(self) -> Iterator: 

433 """Return a context manager that represents a session (persistent 

434 connection to a database). 

435 """ 

436 if self._session_connection is not None: 

437 # session already started, just reuse that 

438 yield Session(self) 

439 else: 

440 # open new connection and close it when done 

441 self._session_connection = self._engine.connect() 

442 yield Session(self) 

443 self._session_connection.close() 

444 self._session_connection = None 

445 # Temporary tables only live within session 

446 self._tempTables = set() 

447 

448 @contextmanager 

449 def transaction(self, *, interrupting: bool = False, savepoint: bool = False, 

450 lock: Iterable[sqlalchemy.schema.Table] = ()) -> Iterator: 

451 """Return a context manager that represents a transaction. 

452 

453 Parameters 

454 ---------- 

455 interrupting : `bool`, optional 

456 If `True` (`False` is default), this transaction block may not be 

457 nested without an outer one, and attempting to do so is a logic 

458 (i.e. assertion) error. 

459 savepoint : `bool`, optional 

460 If `True` (`False` is default), create a `SAVEPOINT`, allowing 

461 exceptions raised by the database (e.g. due to constraint 

462 violations) during this transaction's context to be caught outside 

463 it without also rolling back all operations in an outer transaction 

464 block. If `False`, transactions may still be nested, but a 

465 rollback may be generated at any level and affects all levels, and 

466 commits are deferred until the outermost block completes. If any 

467 outer transaction block was created with ``savepoint=True``, all 

468 inner blocks will be as well (regardless of the actual value 

469 passed). This has no effect if this is the outermost transaction. 

470 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

471 A list of tables to lock for the duration of this transaction. 

472 These locks are guaranteed to prevent concurrent writes and allow 

473 this transaction (only) to acquire the same locks (others should 

474 block), but only prevent concurrent reads if the database engine 

475 requires that in order to block concurrent writes. 

476 

477 Notes 

478 ----- 

479 All transactions on a connection managed by one or more `Database` 

480 instances _must_ go through this method, or transaction state will not 

481 be correctly managed. 

482 """ 

483 # need a connection, use session to manage it 

484 with self.session(): 

485 assert self._session_connection is not None 

486 connection = self._session_connection 

487 assert not (interrupting and connection.in_transaction()), ( 

488 "Logic error in transaction nesting: an operation that would " 

489 "interrupt the active transaction context has been requested." 

490 ) 

491 # We remember whether we are already in a SAVEPOINT transaction via 

492 # the connection object's 'info' dict, which is explicitly for user 

493 # information like this. This is safer than a regular `Database` 

494 # instance attribute, because it guards against multiple `Database` 

495 # instances sharing the same connection. The need to use our own 

496 # flag here to track whether we're in a nested transaction should 

497 # go away in SQLAlchemy 1.4, which seems to have a 

498 # `Connection.in_nested_transaction()` method. 

499 savepoint = savepoint or connection.info.get(_IN_SAVEPOINT_TRANSACTION, False) 

500 connection.info[_IN_SAVEPOINT_TRANSACTION] = savepoint 

501 if connection.in_transaction() and savepoint: 

502 trans = connection.begin_nested() 

503 else: 

504 # Use a regular (non-savepoint) transaction always for the 

505 # outermost context, as well as when a savepoint was not 

506 # requested. 

507 trans = connection.begin() 

508 self._lockTables(lock) 

509 try: 

510 yield 

511 trans.commit() 

512 except BaseException: 

513 trans.rollback() 

514 raise 

515 finally: 

516 if not connection.in_transaction(): 

517 connection.info.pop(_IN_SAVEPOINT_TRANSACTION, None) 

518 

519 @property 

520 def _connection(self) -> sqlalchemy.engine.Connectable: 

521 """Object that can be used to execute queries 

522 (`sqlalchemy.engine.Connectable`) 

523 """ 

524 return self._session_connection or self._engine 

525 

526 @abstractmethod 

527 def _lockTables(self, tables: Iterable[sqlalchemy.schema.Table] = ()) -> None: 

528 """Acquire locks on the given tables. 

529 

530 This is an implementation hook for subclasses, called by `transaction`. 

531 It should not be called directly by other code. 

532 

533 Parameters 

534 ---------- 

535 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

536 A list of tables to lock for the duration of this transaction. 

537 These locks are guaranteed to prevent concurrent writes and allow 

538 this transaction (only) to acquire the same locks (others should 

539 block), but only prevent concurrent reads if the database engine 

540 requires that in order to block concurrent writes. 

541 """ 

542 raise NotImplementedError() 

543 

544 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool: 

545 """Check whether a table is writeable, either because the database 

546 connection is read-write or the table is a temporary table. 

547 

548 Parameters 

549 ---------- 

550 table : `sqlalchemy.schema.Table` 

551 SQLAlchemy table object to check. 

552 

553 Returns 

554 ------- 

555 writeable : `bool` 

556 Whether this table is writeable. 

557 """ 

558 return self.isWriteable() or table.key in self._tempTables 

559 

560 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None: 

561 """Raise if the given table is not writeable, either because the 

562 database connection is read-write or the table is a temporary table. 

563 

564 Parameters 

565 ---------- 

566 table : `sqlalchemy.schema.Table` 

567 SQLAlchemy table object to check. 

568 msg : `str`, optional 

569 If provided, raise `ReadOnlyDatabaseError` instead of returning 

570 `False`, with this message. 

571 """ 

572 if not self.isTableWriteable(table): 

573 raise ReadOnlyDatabaseError(msg) 

574 

575 @contextmanager 

576 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]: 

577 """Return a context manager in which the database's static DDL schema 

578 can be declared. 

579 

580 Parameters 

581 ---------- 

582 create : `bool` 

583 If `True`, attempt to create all tables at the end of the context. 

584 If `False`, they will be assumed to already exist. 

585 

586 Returns 

587 ------- 

588 schema : `StaticTablesContext` 

589 A helper object that is used to add new tables. 

590 

591 Raises 

592 ------ 

593 ReadOnlyDatabaseError 

594 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

595 and one or more declared tables do not already exist. 

596 

597 Examples 

598 -------- 

599 Given a `Database` instance ``db``:: 

600 

601 with db.declareStaticTables(create=True) as schema: 

602 schema.addTable("table1", TableSpec(...)) 

603 schema.addTable("table2", TableSpec(...)) 

604 

605 Notes 

606 ----- 

607 A database's static DDL schema must be declared before any dynamic 

608 tables are managed via calls to `ensureTableExists` or 

609 `getExistingTable`. The order in which static schema tables are added 

610 inside the context block is unimportant; they will automatically be 

611 sorted and added in an order consistent with their foreign key 

612 relationships. 

613 """ 

614 if create and not self.isWriteable(): 

615 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

616 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

617 try: 

618 context = StaticTablesContext(self) 

619 if create and context._tableNames: 

620 # Looks like database is already initalized, to avoid danger 

621 # of modifying/destroying valid schema we refuse to do 

622 # anything in this case 

623 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.") 

624 yield context 

625 for table, foreignKey in context._foreignKeys: 

626 table.append_constraint(foreignKey) 

627 if create: 

628 if self.namespace is not None: 

629 if self.namespace not in context._inspector.get_schema_names(): 

630 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

631 # In our tables we have columns that make use of sqlalchemy 

632 # Sequence objects. There is currently a bug in sqlalchemy that 

633 # causes a deprecation warning to be thrown on a property of 

634 # the Sequence object when the repr for the sequence is 

635 # created. Here a filter is used to catch these deprecation 

636 # warnings when tables are created. 

637 with warnings.catch_warnings(): 

638 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

639 self._metadata.create_all(self._connection) 

640 # call all initializer methods sequentially 

641 for init in context._initializers: 

642 init(self) 

643 except BaseException: 

644 self._metadata = None 

645 raise 

646 

647 @abstractmethod 

648 def isWriteable(self) -> bool: 

649 """Return `True` if this database can be modified by this client. 

650 """ 

651 raise NotImplementedError() 

652 

653 @abstractmethod 

654 def __str__(self) -> str: 

655 """Return a human-readable identifier for this `Database`, including 

656 any namespace or schema that identifies its names within a `Registry`. 

657 """ 

658 raise NotImplementedError() 

659 

660 @property 

661 def dialect(self) -> sqlalchemy.engine.Dialect: 

662 """The SQLAlchemy dialect for this database engine 

663 (`sqlalchemy.engine.Dialect`). 

664 """ 

665 return self._engine.dialect 

666 

667 def shrinkDatabaseEntityName(self, original: str) -> str: 

668 """Return a version of the given name that fits within this database 

669 engine's length limits for table, constraint, indexes, and sequence 

670 names. 

671 

672 Implementations should not assume that simple truncation is safe, 

673 because multiple long names often begin with the same prefix. 

674 

675 The default implementation simply returns the given name. 

676 

677 Parameters 

678 ---------- 

679 original : `str` 

680 The original name. 

681 

682 Returns 

683 ------- 

684 shrunk : `str` 

685 The new, possibly shortened name. 

686 """ 

687 return original 

688 

689 def expandDatabaseEntityName(self, shrunk: str) -> str: 

690 """Retrieve the original name for a database entity that was too long 

691 to fit within the database engine's limits. 

692 

693 Parameters 

694 ---------- 

695 original : `str` 

696 The original name. 

697 

698 Returns 

699 ------- 

700 shrunk : `str` 

701 The new, possibly shortened name. 

702 """ 

703 return shrunk 

704 

705 def _mangleTableName(self, name: str) -> str: 

706 """Map a logical, user-visible table name to the true table name used 

707 in the database. 

708 

709 The default implementation returns the given name unchanged. 

710 

711 Parameters 

712 ---------- 

713 name : `str` 

714 Input table name. Should not include a namespace (i.e. schema) 

715 prefix. 

716 

717 Returns 

718 ------- 

719 mangled : `str` 

720 Mangled version of the table name (still with no namespace prefix). 

721 

722 Notes 

723 ----- 

724 Reimplementations of this method must be idempotent - mangling an 

725 already-mangled name must have no effect. 

726 """ 

727 return name 

728 

729 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]: 

730 """Create constraints based on this spec. 

731 

732 Parameters 

733 ---------- 

734 table : `str` 

735 Name of the table this column is being added to. 

736 spec : `FieldSpec` 

737 Specification for the field to be added. 

738 

739 Returns 

740 ------- 

741 constraint : `list` of `sqlalchemy.CheckConstraint` 

742 Constraint added for this column. 

743 """ 

744 # By default we return no additional constraints 

745 return [] 

746 

747 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

748 **kwds: Any) -> sqlalchemy.schema.Column: 

749 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

750 

751 Parameters 

752 ---------- 

753 table : `str` 

754 Name of the table this column is being added to. 

755 spec : `FieldSpec` 

756 Specification for the field to be added. 

757 metadata : `sqlalchemy.MetaData` 

758 SQLAlchemy representation of the DDL schema this field's table is 

759 being added to. 

760 **kwds 

761 Additional keyword arguments to forward to the 

762 `sqlalchemy.schema.Column` constructor. This is provided to make 

763 it easier for derived classes to delegate to ``super()`` while 

764 making only minor changes. 

765 

766 Returns 

767 ------- 

768 column : `sqlalchemy.schema.Column` 

769 SQLAlchemy representation of the field. 

770 """ 

771 args = [spec.name, spec.getSizedColumnType()] 

772 if spec.autoincrement: 

773 # Generate a sequence to use for auto incrementing for databases 

774 # that do not support it natively. This will be ignored by 

775 # sqlalchemy for databases that do support it. 

776 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

777 metadata=metadata)) 

778 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

779 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

780 comment=spec.doc, server_default=spec.default, **kwds) 

781 

782 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

783 **kwds: Any) -> sqlalchemy.schema.ForeignKeyConstraint: 

784 """Convert a `ForeignKeySpec` to a 

785 `sqlalchemy.schema.ForeignKeyConstraint`. 

786 

787 Parameters 

788 ---------- 

789 table : `str` 

790 Name of the table this foreign key is being added to. 

791 spec : `ForeignKeySpec` 

792 Specification for the foreign key to be added. 

793 metadata : `sqlalchemy.MetaData` 

794 SQLAlchemy representation of the DDL schema this constraint is 

795 being added to. 

796 **kwds 

797 Additional keyword arguments to forward to the 

798 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

799 provided to make it easier for derived classes to delegate to 

800 ``super()`` while making only minor changes. 

801 

802 Returns 

803 ------- 

804 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

805 SQLAlchemy representation of the constraint. 

806 """ 

807 name = self.shrinkDatabaseEntityName( 

808 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

809 + list(spec.target) + list(spec.source)) 

810 ) 

811 return sqlalchemy.schema.ForeignKeyConstraint( 

812 spec.source, 

813 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

814 name=name, 

815 ondelete=spec.onDelete 

816 ) 

817 

818 def _convertExclusionConstraintSpec(self, table: str, 

819 spec: Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...], 

820 metadata: sqlalchemy.MetaData) -> sqlalchemy.schema.Constraint: 

821 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy 

822 constraint representation. 

823 

824 Parameters 

825 ---------- 

826 table : `str` 

827 Name of the table this constraint is being added to. 

828 spec : `tuple` [ `str` or `type` ] 

829 A tuple of `str` column names and the `type` object returned by 

830 `getTimespanRepresentation` (which must appear exactly once), 

831 indicating the order of the columns in the index used to back the 

832 constraint. 

833 metadata : `sqlalchemy.MetaData` 

834 SQLAlchemy representation of the DDL schema this constraint is 

835 being added to. 

836 

837 Returns 

838 ------- 

839 constraint : `sqlalchemy.schema.Constraint` 

840 SQLAlchemy representation of the constraint. 

841 

842 Raises 

843 ------ 

844 NotImplementedError 

845 Raised if this database does not support exclusion constraints. 

846 """ 

847 raise NotImplementedError(f"Database {self} does not support exclusion constraints.") 

848 

849 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

850 **kwds: Any) -> sqlalchemy.schema.Table: 

851 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

852 

853 Parameters 

854 ---------- 

855 spec : `TableSpec` 

856 Specification for the foreign key to be added. 

857 metadata : `sqlalchemy.MetaData` 

858 SQLAlchemy representation of the DDL schema this table is being 

859 added to. 

860 **kwds 

861 Additional keyword arguments to forward to the 

862 `sqlalchemy.schema.Table` constructor. This is provided to make it 

863 easier for derived classes to delegate to ``super()`` while making 

864 only minor changes. 

865 

866 Returns 

867 ------- 

868 table : `sqlalchemy.schema.Table` 

869 SQLAlchemy representation of the table. 

870 

871 Notes 

872 ----- 

873 This method does not handle ``spec.foreignKeys`` at all, in order to 

874 avoid circular dependencies. These are added by higher-level logic in 

875 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

876 """ 

877 name = self._mangleTableName(name) 

878 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

879 

880 # Add any column constraints 

881 for fieldSpec in spec.fields: 

882 args.extend(self._makeColumnConstraints(name, fieldSpec)) 

883 

884 # Track indexes added for primary key and unique constraints, to make 

885 # sure we don't add duplicate explicit or foreign key indexes for 

886 # those. 

887 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)} 

888 args.extend( 

889 sqlalchemy.schema.UniqueConstraint( 

890 *columns, 

891 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

892 ) 

893 for columns in spec.unique 

894 ) 

895 allIndexes.update(spec.unique) 

896 args.extend( 

897 sqlalchemy.schema.Index( 

898 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

899 *columns, 

900 unique=(columns in spec.unique) 

901 ) 

902 for columns in spec.indexes if columns not in allIndexes 

903 ) 

904 allIndexes.update(spec.indexes) 

905 args.extend( 

906 sqlalchemy.schema.Index( 

907 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)), 

908 *fk.source, 

909 ) 

910 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes 

911 ) 

912 

913 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion) 

914 

915 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

916 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

917 

918 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

919 """Ensure that a table with the given name and specification exists, 

920 creating it if necessary. 

921 

922 Parameters 

923 ---------- 

924 name : `str` 

925 Name of the table (not including namespace qualifiers). 

926 spec : `TableSpec` 

927 Specification for the table. This will be used when creating the 

928 table, and *may* be used when obtaining an existing table to check 

929 for consistency, but no such check is guaranteed. 

930 

931 Returns 

932 ------- 

933 table : `sqlalchemy.schema.Table` 

934 SQLAlchemy representation of the table. 

935 

936 Raises 

937 ------ 

938 ReadOnlyDatabaseError 

939 Raised if `isWriteable` returns `False`, and the table does not 

940 already exist. 

941 DatabaseConflictError 

942 Raised if the table exists but ``spec`` is inconsistent with its 

943 definition. 

944 

945 Notes 

946 ----- 

947 This method may not be called within transactions. It may be called on 

948 read-only databases if and only if the table does in fact already 

949 exist. 

950 

951 Subclasses may override this method, but usually should not need to. 

952 """ 

953 # TODO: if _engine is used to make a table then it uses separate 

954 # connection and should not interfere with current transaction 

955 assert self._session_connection is None or not self._session_connection.in_transaction(), \ 

956 "Table creation interrupts transactions." 

957 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

958 table = self.getExistingTable(name, spec) 

959 if table is not None: 

960 return table 

961 if not self.isWriteable(): 

962 raise ReadOnlyDatabaseError( 

963 f"Table {name} does not exist, and cannot be created " 

964 f"because database {self} is read-only." 

965 ) 

966 table = self._convertTableSpec(name, spec, self._metadata) 

967 for foreignKeySpec in spec.foreignKeys: 

968 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

969 table.create(self._connection) 

970 return table 

971 

972 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

973 """Obtain an existing table with the given name and specification. 

974 

975 Parameters 

976 ---------- 

977 name : `str` 

978 Name of the table (not including namespace qualifiers). 

979 spec : `TableSpec` 

980 Specification for the table. This will be used when creating the 

981 SQLAlchemy representation of the table, and it is used to 

982 check that the actual table in the database is consistent. 

983 

984 Returns 

985 ------- 

986 table : `sqlalchemy.schema.Table` or `None` 

987 SQLAlchemy representation of the table, or `None` if it does not 

988 exist. 

989 

990 Raises 

991 ------ 

992 DatabaseConflictError 

993 Raised if the table exists but ``spec`` is inconsistent with its 

994 definition. 

995 

996 Notes 

997 ----- 

998 This method can be called within transactions and never modifies the 

999 database. 

1000 

1001 Subclasses may override this method, but usually should not need to. 

1002 """ 

1003 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

1004 name = self._mangleTableName(name) 

1005 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

1006 if table is not None: 

1007 if spec.fields.names != set(table.columns.keys()): 

1008 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

1009 f"specification has columns {list(spec.fields.names)}, while " 

1010 f"the previous definition has {list(table.columns.keys())}.") 

1011 else: 

1012 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

1013 if name in inspector.get_table_names(schema=self.namespace): 

1014 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

1015 table = self._convertTableSpec(name, spec, self._metadata) 

1016 for foreignKeySpec in spec.foreignKeys: 

1017 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

1018 return table 

1019 return table 

1020 

1021 @classmethod 

1022 def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]: 

1023 """Return a `type` that encapsulates the way `Timespan` objects are 

1024 stored in this database. 

1025 

1026 `Database` does not automatically use the return type of this method 

1027 anywhere else; calling code is responsible for making sure that DDL 

1028 and queries are consistent with it. 

1029 

1030 Returns 

1031 ------- 

1032 TimespanReprClass : `type` (`TimespanDatabaseRepresention` subclass) 

1033 A type that encapsulates the way `Timespan` objects should be 

1034 stored in this database. 

1035 

1036 Notes 

1037 ----- 

1038 There are two big reasons we've decided to keep timespan-mangling logic 

1039 outside the `Database` implementations, even though the choice of 

1040 representation is ultimately up to a `Database` implementation: 

1041 

1042 - Timespans appear in relatively few tables and queries in our 

1043 typical usage, and the code that operates on them is already aware 

1044 that it is working with timespans. In contrast, a 

1045 timespan-representation-aware implementation of, say, `insert`, 

1046 would need to have extra logic to identify when timespan-mangling 

1047 needed to occur, which would usually be useless overhead. 

1048 

1049 - SQLAlchemy's rich SELECT query expression system has no way to wrap 

1050 multiple columns in a single expression object (the ORM does, but 

1051 we are not using the ORM). So we would have to wrap _much_ more of 

1052 that code in our own interfaces to encapsulate timespan 

1053 representations there. 

1054 """ 

1055 return TimespanDatabaseRepresentation.Compound 

1056 

1057 @classmethod 

1058 def getSpatialRegionRepresentation(cls) -> Type[SpatialRegionDatabaseRepresentation]: 

1059 """Return a `type` that encapsulates the way `lsst.sphgeom.Region` 

1060 objects are stored in this database. 

1061 

1062 `Database` does not automatically use the return type of this method 

1063 anywhere else; calling code is responsible for making sure that DDL 

1064 and queries are consistent with it. 

1065 

1066 Returns 

1067 ------- 

1068 RegionReprClass : `type` (`SpatialRegionDatabaseRepresention` subclass) 

1069 A type that encapsulates the way `lsst.sphgeom.Region` objects 

1070 should be stored in this database. 

1071 

1072 Notes 

1073 ----- 

1074 See `getTimespanRepresentation` for comments on why this method is not 

1075 more tightly integrated with the rest of the `Database` interface. 

1076 """ 

1077 return SpatialRegionDatabaseRepresentation 

1078 

1079 def sync(self, table: sqlalchemy.schema.Table, *, 

1080 keys: Dict[str, Any], 

1081 compared: Optional[Dict[str, Any]] = None, 

1082 extra: Optional[Dict[str, Any]] = None, 

1083 returning: Optional[Sequence[str]] = None, 

1084 ) -> Tuple[Optional[Dict[str, Any]], bool]: 

1085 """Insert into a table as necessary to ensure database contains 

1086 values equivalent to the given ones. 

1087 

1088 Parameters 

1089 ---------- 

1090 table : `sqlalchemy.schema.Table` 

1091 Table to be queried and possibly inserted into. 

1092 keys : `dict` 

1093 Column name-value pairs used to search for an existing row; must 

1094 be a combination that can be used to select a single row if one 

1095 exists. If such a row does not exist, these values are used in 

1096 the insert. 

1097 compared : `dict`, optional 

1098 Column name-value pairs that are compared to those in any existing 

1099 row. If such a row does not exist, these rows are used in the 

1100 insert. 

1101 extra : `dict`, optional 

1102 Column name-value pairs that are ignored if a matching row exists, 

1103 but used in an insert if one is necessary. 

1104 returning : `~collections.abc.Sequence` of `str`, optional 

1105 The names of columns whose values should be returned. 

1106 

1107 Returns 

1108 ------- 

1109 row : `dict`, optional 

1110 The value of the fields indicated by ``returning``, or `None` if 

1111 ``returning`` is `None`. 

1112 inserted : `bool` 

1113 If `True`, a new row was inserted. 

1114 

1115 Raises 

1116 ------ 

1117 DatabaseConflictError 

1118 Raised if the values in ``compared`` do not match the values in the 

1119 database. 

1120 ReadOnlyDatabaseError 

1121 Raised if `isWriteable` returns `False`, and no matching record 

1122 already exists. 

1123 

1124 Notes 

1125 ----- 

1126 May be used inside transaction contexts, so implementations may not 

1127 perform operations that interrupt transactions. 

1128 

1129 It may be called on read-only databases if and only if the matching row 

1130 does in fact already exist. 

1131 """ 

1132 

1133 def check() -> Tuple[int, Optional[List[str]], Optional[List]]: 

1134 """Query for a row that matches the ``key`` argument, and compare 

1135 to what was given by the caller. 

1136 

1137 Returns 

1138 ------- 

1139 n : `int` 

1140 Number of matching rows. ``n != 1`` is always an error, but 

1141 it's a different kind of error depending on where `check` is 

1142 being called. 

1143 bad : `list` of `str`, or `None` 

1144 The subset of the keys of ``compared`` for which the existing 

1145 values did not match the given one. Once again, ``not bad`` 

1146 is always an error, but a different kind on context. `None` 

1147 if ``n != 1`` 

1148 result : `list` or `None` 

1149 Results in the database that correspond to the columns given 

1150 in ``returning``, or `None` if ``returning is None``. 

1151 """ 

1152 toSelect: Set[str] = set() 

1153 if compared is not None: 

1154 toSelect.update(compared.keys()) 

1155 if returning is not None: 

1156 toSelect.update(returning) 

1157 if not toSelect: 

1158 # Need to select some column, even if we just want to see 

1159 # how many rows we get back. 

1160 toSelect.add(next(iter(keys.keys()))) 

1161 selectSql = sqlalchemy.sql.select( 

1162 [table.columns[k].label(k) for k in toSelect] 

1163 ).select_from(table).where( 

1164 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

1165 ) 

1166 fetched = list(self._connection.execute(selectSql).fetchall()) 

1167 if len(fetched) != 1: 

1168 return len(fetched), None, None 

1169 existing = fetched[0] 

1170 if compared is not None: 

1171 

1172 def safeNotEqual(a: Any, b: Any) -> bool: 

1173 if isinstance(a, astropy.time.Time): 

1174 return not time_utils.TimeConverter().times_equal(a, b) 

1175 return a != b 

1176 

1177 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}" 

1178 for k, v in compared.items() 

1179 if safeNotEqual(existing[k], v)] 

1180 else: 

1181 inconsistencies = [] 

1182 if returning is not None: 

1183 toReturn: Optional[list] = [existing[k] for k in returning] 

1184 else: 

1185 toReturn = None 

1186 return 1, inconsistencies, toReturn 

1187 

1188 if self.isTableWriteable(table): 

1189 # Try an insert first, but allow it to fail (in only specific 

1190 # ways). 

1191 row = keys.copy() 

1192 if compared is not None: 

1193 row.update(compared) 

1194 if extra is not None: 

1195 row.update(extra) 

1196 with self.transaction(lock=[table]): 

1197 inserted = bool(self.ensure(table, row)) 

1198 # Need to perform check() for this branch inside the 

1199 # transaction, so we roll back an insert that didn't do 

1200 # what we expected. That limits the extent to which we 

1201 # can reduce duplication between this block and the other 

1202 # ones that perform similar logic. 

1203 n, bad, result = check() 

1204 if n < 1: 

1205 raise ConflictingDefinitionError( 

1206 f"Attempted to ensure {row} exists by inserting it with ON CONFLICT IGNORE, " 

1207 f"but a post-insert query on {keys} returned no results. " 

1208 f"Insert was {'' if inserted else 'not '}reported as successful. " 

1209 "This can occur if the insert violated a database constraint other than the " 

1210 "unique constraint or primary key used to identify the row in this call." 

1211 ) 

1212 elif n > 1: 

1213 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

1214 f"unique constraint for table {table.name}.") 

1215 elif bad: 

1216 if inserted: 

1217 raise RuntimeError( 

1218 f"Conflict ({bad}) in sync after successful insert; this is " 

1219 "possible if the same table is being updated by a concurrent " 

1220 "process that isn't using sync, but it may also be a bug in " 

1221 "daf_butler." 

1222 ) 

1223 else: 

1224 raise DatabaseConflictError( 

1225 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1226 ) 

1227 else: 

1228 # Database is not writeable; just see if the row exists. 

1229 n, bad, result = check() 

1230 if n < 1: 

1231 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

1232 elif n > 1: 

1233 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

1234 elif bad: 

1235 raise DatabaseConflictError( 

1236 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1237 ) 

1238 inserted = False 

1239 if returning is None: 

1240 return None, inserted 

1241 else: 

1242 assert result is not None 

1243 return {k: v for k, v in zip(returning, result)}, inserted 

1244 

1245 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

1246 select: Optional[sqlalchemy.sql.Select] = None, 

1247 names: Optional[Iterable[str]] = None, 

1248 ) -> Optional[List[int]]: 

1249 """Insert one or more rows into a table, optionally returning 

1250 autoincrement primary key values. 

1251 

1252 Parameters 

1253 ---------- 

1254 table : `sqlalchemy.schema.Table` 

1255 Table rows should be inserted into. 

1256 returnIds: `bool` 

1257 If `True` (`False` is default), return the values of the table's 

1258 autoincrement primary key field (which much exist). 

1259 select : `sqlalchemy.sql.Select`, optional 

1260 A SELECT query expression to insert rows from. Cannot be provided 

1261 with either ``rows`` or ``returnIds=True``. 

1262 names : `Iterable` [ `str` ], optional 

1263 Names of columns in ``table`` to be populated, ordered to match the 

1264 columns returned by ``select``. Ignored if ``select`` is `None`. 

1265 If not provided, the columns returned by ``select`` must be named 

1266 to match the desired columns of ``table``. 

1267 *rows 

1268 Positional arguments are the rows to be inserted, as dictionaries 

1269 mapping column name to value. The keys in all dictionaries must 

1270 be the same. 

1271 

1272 Returns 

1273 ------- 

1274 ids : `None`, or `list` of `int` 

1275 If ``returnIds`` is `True`, a `list` containing the inserted 

1276 values for the table's autoincrement primary key. 

1277 

1278 Raises 

1279 ------ 

1280 ReadOnlyDatabaseError 

1281 Raised if `isWriteable` returns `False` when this method is called. 

1282 

1283 Notes 

1284 ----- 

1285 The default implementation uses bulk insert syntax when ``returnIds`` 

1286 is `False`, and a loop over single-row insert operations when it is 

1287 `True`. 

1288 

1289 Derived classes should reimplement when they can provide a more 

1290 efficient implementation (especially for the latter case). 

1291 

1292 May be used inside transaction contexts, so implementations may not 

1293 perform operations that interrupt transactions. 

1294 """ 

1295 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.") 

1296 if select is not None and (rows or returnIds): 

1297 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.") 

1298 if not rows and select is None: 

1299 if returnIds: 

1300 return [] 

1301 else: 

1302 return None 

1303 if not returnIds: 

1304 if select is not None: 

1305 if names is None: 

1306 names = select.columns.keys() 

1307 self._connection.execute(table.insert().from_select(names, select)) 

1308 else: 

1309 self._connection.execute(table.insert(), *rows) 

1310 return None 

1311 else: 

1312 sql = table.insert() 

1313 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

1314 

1315 @abstractmethod 

1316 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None: 

1317 """Insert one or more rows into a table, replacing any existing rows 

1318 for which insertion of a new row would violate the primary key 

1319 constraint. 

1320 

1321 Parameters 

1322 ---------- 

1323 table : `sqlalchemy.schema.Table` 

1324 Table rows should be inserted into. 

1325 *rows 

1326 Positional arguments are the rows to be inserted, as dictionaries 

1327 mapping column name to value. The keys in all dictionaries must 

1328 be the same. 

1329 

1330 Raises 

1331 ------ 

1332 ReadOnlyDatabaseError 

1333 Raised if `isWriteable` returns `False` when this method is called. 

1334 

1335 Notes 

1336 ----- 

1337 May be used inside transaction contexts, so implementations may not 

1338 perform operations that interrupt transactions. 

1339 

1340 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

1341 exception when a constraint other than the primary key would be 

1342 violated. 

1343 

1344 Implementations are not required to support `replace` on tables 

1345 with autoincrement keys. 

1346 """ 

1347 raise NotImplementedError() 

1348 

1349 @abstractmethod 

1350 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict) -> int: 

1351 """Insert one or more rows into a table, skipping any rows for which 

1352 insertion would violate any constraint. 

1353 

1354 Parameters 

1355 ---------- 

1356 table : `sqlalchemy.schema.Table` 

1357 Table rows should be inserted into. 

1358 *rows 

1359 Positional arguments are the rows to be inserted, as dictionaries 

1360 mapping column name to value. The keys in all dictionaries must 

1361 be the same. 

1362 

1363 Returns 

1364 ------- 

1365 count : `int` 

1366 The number of rows actually inserted. 

1367 

1368 Raises 

1369 ------ 

1370 ReadOnlyDatabaseError 

1371 Raised if `isWriteable` returns `False` when this method is called. 

1372 This is raised even if the operation would do nothing even on a 

1373 writeable database. 

1374 

1375 Notes 

1376 ----- 

1377 May be used inside transaction contexts, so implementations may not 

1378 perform operations that interrupt transactions. 

1379 

1380 Implementations are not required to support `ensure` on tables 

1381 with autoincrement keys. 

1382 """ 

1383 raise NotImplementedError() 

1384 

1385 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

1386 """Delete one or more rows from a table. 

1387 

1388 Parameters 

1389 ---------- 

1390 table : `sqlalchemy.schema.Table` 

1391 Table that rows should be deleted from. 

1392 columns: `~collections.abc.Iterable` of `str` 

1393 The names of columns that will be used to constrain the rows to 

1394 be deleted; these will be combined via ``AND`` to form the 

1395 ``WHERE`` clause of the delete query. 

1396 *rows 

1397 Positional arguments are the keys of rows to be deleted, as 

1398 dictionaries mapping column name to value. The keys in all 

1399 dictionaries must be exactly the names in ``columns``. 

1400 

1401 Returns 

1402 ------- 

1403 count : `int` 

1404 Number of rows deleted. 

1405 

1406 Raises 

1407 ------ 

1408 ReadOnlyDatabaseError 

1409 Raised if `isWriteable` returns `False` when this method is called. 

1410 

1411 Notes 

1412 ----- 

1413 May be used inside transaction contexts, so implementations may not 

1414 perform operations that interrupt transactions. 

1415 

1416 The default implementation should be sufficient for most derived 

1417 classes. 

1418 """ 

1419 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.") 

1420 if columns and not rows: 

1421 # If there are no columns, this operation is supposed to delete 

1422 # everything (so we proceed as usual). But if there are columns, 

1423 # but no rows, it was a constrained bulk operation where the 

1424 # constraint is that no rows match, and we should short-circuit 

1425 # while reporting that no rows were affected. 

1426 return 0 

1427 sql = table.delete() 

1428 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

1429 if whereTerms: 

1430 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

1431 return self._connection.execute(sql, *rows).rowcount 

1432 

1433 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

1434 """Update one or more rows in a table. 

1435 

1436 Parameters 

1437 ---------- 

1438 table : `sqlalchemy.schema.Table` 

1439 Table containing the rows to be updated. 

1440 where : `dict` [`str`, `str`] 

1441 A mapping from the names of columns that will be used to search for 

1442 existing rows to the keys that will hold these values in the 

1443 ``rows`` dictionaries. Note that these may not be the same due to 

1444 SQLAlchemy limitations. 

1445 *rows 

1446 Positional arguments are the rows to be updated. The keys in all 

1447 dictionaries must be the same, and may correspond to either a 

1448 value in the ``where`` dictionary or the name of a column to be 

1449 updated. 

1450 

1451 Returns 

1452 ------- 

1453 count : `int` 

1454 Number of rows matched (regardless of whether the update actually 

1455 modified them). 

1456 

1457 Raises 

1458 ------ 

1459 ReadOnlyDatabaseError 

1460 Raised if `isWriteable` returns `False` when this method is called. 

1461 

1462 Notes 

1463 ----- 

1464 May be used inside transaction contexts, so implementations may not 

1465 perform operations that interrupt transactions. 

1466 

1467 The default implementation should be sufficient for most derived 

1468 classes. 

1469 """ 

1470 self.assertTableWriteable(table, f"Cannot update read-only table {table}.") 

1471 if not rows: 

1472 return 0 

1473 sql = table.update().where( 

1474 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1475 ) 

1476 return self._connection.execute(sql, *rows).rowcount 

1477 

1478 def query(self, sql: sqlalchemy.sql.FromClause, 

1479 *args: Any, **kwds: Any) -> sqlalchemy.engine.ResultProxy: 

1480 """Run a SELECT query against the database. 

1481 

1482 Parameters 

1483 ---------- 

1484 sql : `sqlalchemy.sql.FromClause` 

1485 A SQLAlchemy representation of a ``SELECT`` query. 

1486 *args 

1487 Additional positional arguments are forwarded to 

1488 `sqlalchemy.engine.Connection.execute`. 

1489 **kwds 

1490 Additional keyword arguments are forwarded to 

1491 `sqlalchemy.engine.Connection.execute`. 

1492 

1493 Returns 

1494 ------- 

1495 result : `sqlalchemy.engine.ResultProxy` 

1496 Query results. 

1497 

1498 Notes 

1499 ----- 

1500 The default implementation should be sufficient for most derived 

1501 classes. 

1502 """ 

1503 # TODO: should we guard against non-SELECT queries here? 

1504 return self._connection.execute(sql, *args, **kwds) 

1505 

1506 origin: int 

1507 """An integer ID that should be used as the default for any datasets, 

1508 quanta, or other entities that use a (autoincrement, origin) compound 

1509 primary key (`int`). 

1510 """ 

1511 

1512 namespace: Optional[str] 

1513 """The schema or namespace this database instance is associated with 

1514 (`str` or `None`). 

1515 """