Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "SchemaAlreadyDefinedError", 

28 "StaticTablesContext", 

29] 

30 

31from abc import ABC, abstractmethod 

32from collections import defaultdict 

33from contextlib import contextmanager 

34from typing import ( 

35 Any, 

36 Callable, 

37 Dict, 

38 Iterable, 

39 Iterator, 

40 List, 

41 Optional, 

42 Sequence, 

43 Set, 

44 Tuple, 

45 Type, 

46 Union, 

47) 

48import uuid 

49import warnings 

50 

51import astropy.time 

52import sqlalchemy 

53 

54from ...core import SpatialRegionDatabaseRepresentation, TimespanDatabaseRepresentation, ddl, time_utils 

55from .._exceptions import ConflictingDefinitionError 

56 

57_IN_SAVEPOINT_TRANSACTION = "IN_SAVEPOINT_TRANSACTION" 

58 

59 

60def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None: 

61 """Test that the definition of a table in a `ddl.TableSpec` and from 

62 database introspection are consistent. 

63 

64 Parameters 

65 ---------- 

66 name : `str` 

67 Name of the table (only used in error messages). 

68 spec : `ddl.TableSpec` 

69 Specification of the table. 

70 inspection : `dict` 

71 Dictionary returned by 

72 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

73 

74 Raises 

75 ------ 

76 DatabaseConflictError 

77 Raised if the definitions are inconsistent. 

78 """ 

79 columnNames = [c["name"] for c in inspection] 

80 if spec.fields.names != set(columnNames): 

81 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

82 f"specification has columns {list(spec.fields.names)}, while the " 

83 f"table in the database has {columnNames}.") 

84 

85 

86class ReadOnlyDatabaseError(RuntimeError): 

87 """Exception raised when a write operation is called on a read-only 

88 `Database`. 

89 """ 

90 

91 

92class DatabaseConflictError(ConflictingDefinitionError): 

93 """Exception raised when database content (row values or schema entities) 

94 are inconsistent with what this client expects. 

95 """ 

96 

97 

98class SchemaAlreadyDefinedError(RuntimeError): 

99 """Exception raised when trying to initialize database schema when some 

100 tables already exist. 

101 """ 

102 

103 

104class StaticTablesContext: 

105 """Helper class used to declare the static schema for a registry layer 

106 in a database. 

107 

108 An instance of this class is returned by `Database.declareStaticTables`, 

109 which should be the only way it should be constructed. 

110 """ 

111 

112 def __init__(self, db: Database): 

113 self._db = db 

114 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = [] 

115 self._inspector = sqlalchemy.inspect(self._db._connection) 

116 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

117 self._initializers: List[Callable[[Database], None]] = [] 

118 

119 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

120 """Add a new table to the schema, returning its sqlalchemy 

121 representation. 

122 

123 The new table may not actually be created until the end of the 

124 context created by `Database.declareStaticTables`, allowing tables 

125 to be declared in any order even in the presence of foreign key 

126 relationships. 

127 """ 

128 name = self._db._mangleTableName(name) 

129 if name in self._tableNames: 

130 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

131 schema=self._db.namespace)) 

132 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

133 for foreignKeySpec in spec.foreignKeys: 

134 self._foreignKeys.append( 

135 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

136 ) 

137 return table 

138 

139 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

140 """Add a named tuple of tables to the schema, returning their 

141 SQLAlchemy representations in a named tuple of the same type. 

142 

143 The new tables may not actually be created until the end of the 

144 context created by `Database.declareStaticTables`, allowing tables 

145 to be declared in any order even in the presence of foreign key 

146 relationships. 

147 

148 Notes 

149 ----- 

150 ``specs`` *must* be an instance of a type created by 

151 `collections.namedtuple`, not just regular tuple, and the returned 

152 object is guaranteed to be the same. Because `~collections.namedtuple` 

153 is just a factory for `type` objects, not an actual type itself, 

154 we cannot represent this with type annotations. 

155 """ 

156 return specs._make(self.addTable(name, spec) # type: ignore 

157 for name, spec in zip(specs._fields, specs)) # type: ignore 

158 

159 def addInitializer(self, initializer: Callable[[Database], None]) -> None: 

160 """Add a method that does one-time initialization of a database. 

161 

162 Initialization can mean anything that changes state of a database 

163 and needs to be done exactly once after database schema was created. 

164 An example for that could be population of schema attributes. 

165 

166 Parameters 

167 ---------- 

168 initializer : callable 

169 Method of a single argument which is a `Database` instance. 

170 """ 

171 self._initializers.append(initializer) 

172 

173 

174class Session: 

175 """Class representing a persistent connection to a database. 

176 

177 Parameters 

178 ---------- 

179 db : `Database` 

180 Database instance. 

181 

182 Notes 

183 ----- 

184 Instances of Session class should not be created by client code; 

185 `Database.session` should be used to create context for a session:: 

186 

187 with db.session() as session: 

188 session.method() 

189 db.method() 

190 

191 In the current implementation sessions can be nested and transactions can 

192 be nested within a session. All nested sessions and transaction share the 

193 same database connection. 

194 

195 Session class represents a limited subset of database API that requires 

196 persistent connection to a database (e.g. temporary tables which have 

197 lifetime of a session). Potentially most of the database API could be 

198 associated with a Session class. 

199 """ 

200 def __init__(self, db: Database): 

201 self._db = db 

202 

203 def makeTemporaryTable(self, spec: ddl.TableSpec, name: Optional[str] = None) -> sqlalchemy.schema.Table: 

204 """Create a temporary table. 

205 

206 Parameters 

207 ---------- 

208 spec : `TableSpec` 

209 Specification for the table. 

210 name : `str`, optional 

211 A unique (within this session/connetion) name for the table. 

212 Subclasses may override to modify the actual name used. If not 

213 provided, a unique name will be generated. 

214 

215 Returns 

216 ------- 

217 table : `sqlalchemy.schema.Table` 

218 SQLAlchemy representation of the table. 

219 

220 Notes 

221 ----- 

222 Temporary tables may be created, dropped, and written to even in 

223 read-only databases - at least according to the Python-level 

224 protections in the `Database` classes. Server permissions may say 

225 otherwise, but in that case they probably need to be modified to 

226 support the full range of expected read-only butler behavior. 

227 

228 Temporary table rows are guaranteed to be dropped when a connection is 

229 closed. `Database` implementations are permitted to allow the table to 

230 remain as long as this is transparent to the user (i.e. "creating" the 

231 temporary table in a new session should not be an error, even if it 

232 does nothing). 

233 

234 It may not be possible to use temporary tables within transactions with 

235 some database engines (or configurations thereof). 

236 """ 

237 if name is None: 

238 name = f"tmp_{uuid.uuid4().hex}" 

239 table = self._db._convertTableSpec(name, spec, self._db._metadata, prefixes=['TEMPORARY'], 

240 schema=sqlalchemy.schema.BLANK_SCHEMA) 

241 if table.key in self._db._tempTables: 

242 if table.key != name: 

243 raise ValueError(f"A temporary table with name {name} (transformed to {table.key} by " 

244 f"Database) already exists.") 

245 for foreignKeySpec in spec.foreignKeys: 

246 table.append_constraint(self._db._convertForeignKeySpec(name, foreignKeySpec, 

247 self._db._metadata)) 

248 table.create(self._db._session_connection) 

249 self._db._tempTables.add(table.key) 

250 return table 

251 

252 def dropTemporaryTable(self, table: sqlalchemy.schema.Table) -> None: 

253 """Drop a temporary table. 

254 

255 Parameters 

256 ---------- 

257 table : `sqlalchemy.schema.Table` 

258 A SQLAlchemy object returned by a previous call to 

259 `makeTemporaryTable`. 

260 """ 

261 if table.key in self._db._tempTables: 

262 table.drop(self._db._session_connection) 

263 self._db._tempTables.remove(table.key) 

264 else: 

265 raise TypeError(f"Table {table.key} was not created by makeTemporaryTable.") 

266 

267 

268class Database(ABC): 

269 """An abstract interface that represents a particular database engine's 

270 representation of a single schema/namespace/database. 

271 

272 Parameters 

273 ---------- 

274 origin : `int` 

275 An integer ID that should be used as the default for any datasets, 

276 quanta, or other entities that use a (autoincrement, origin) compound 

277 primary key. 

278 engine : `sqlalchemy.engine.Engine` 

279 The SQLAlchemy engine for this `Database`. 

280 namespace : `str`, optional 

281 Name of the schema or namespace this instance is associated with. 

282 This is passed as the ``schema`` argument when constructing a 

283 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

284 avoid confusion between "schema means namespace" and "schema means 

285 table definitions". 

286 

287 Notes 

288 ----- 

289 `Database` requires all write operations to go through its special named 

290 methods. Our write patterns are sufficiently simple that we don't really 

291 need the full flexibility of SQL insert/update/delete syntax, and we need 

292 non-standard (but common) functionality in these operations sufficiently 

293 often that it seems worthwhile to provide our own generic API. 

294 

295 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

296 their SQLAlchemy representation) to be run, as we expect these to require 

297 significantly more sophistication while still being limited to standard 

298 SQL. 

299 

300 `Database` itself has several underscore-prefixed attributes: 

301 

302 - ``_engine``: SQLAlchemy object representing its engine. 

303 - ``_connection``: the `sqlalchemy.engine.Connectable` object which can 

304 be either an Engine or Connection if a session is active. 

305 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

306 the tables and other schema entities. 

307 

308 These are considered protected (derived classes may access them, but other 

309 code should not), and read-only, aside from executing SQL via 

310 ``_connection``. 

311 """ 

312 

313 def __init__(self, *, origin: int, engine: sqlalchemy.engine.Engine, 

314 namespace: Optional[str] = None): 

315 self.origin = origin 

316 self.namespace = namespace 

317 self._engine = engine 

318 self._session_connection: Optional[sqlalchemy.engine.Connection] = None 

319 self._metadata: Optional[sqlalchemy.schema.MetaData] = None 

320 self._tempTables: Set[str] = set() 

321 

322 def __repr__(self) -> str: 

323 # Rather than try to reproduce all the parameters used to create 

324 # the object, instead report the more useful information of the 

325 # connection URL. 

326 uri = str(self._engine.url) 

327 if self.namespace: 

328 uri += f"#{self.namespace}" 

329 return f'{type(self).__name__}("{uri}")' 

330 

331 @classmethod 

332 def makeDefaultUri(cls, root: str) -> Optional[str]: 

333 """Create a default connection URI appropriate for the given root 

334 directory, or `None` if there can be no such default. 

335 """ 

336 return None 

337 

338 @classmethod 

339 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

340 writeable: bool = True) -> Database: 

341 """Construct a database from a SQLAlchemy URI. 

342 

343 Parameters 

344 ---------- 

345 uri : `str` 

346 A SQLAlchemy URI connection string. 

347 origin : `int` 

348 An integer ID that should be used as the default for any datasets, 

349 quanta, or other entities that use a (autoincrement, origin) 

350 compound primary key. 

351 namespace : `str`, optional 

352 A database namespace (i.e. schema) the new instance should be 

353 associated with. If `None` (default), the namespace (if any) is 

354 inferred from the URI. 

355 writeable : `bool`, optional 

356 If `True`, allow write operations on the database, including 

357 ``CREATE TABLE``. 

358 

359 Returns 

360 ------- 

361 db : `Database` 

362 A new `Database` instance. 

363 """ 

364 return cls.fromEngine(cls.makeEngine(uri, writeable=writeable), 

365 origin=origin, 

366 namespace=namespace, 

367 writeable=writeable) 

368 

369 @classmethod 

370 @abstractmethod 

371 def makeEngine(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Engine: 

372 """Create a `sqlalchemy.engine.Engine` from a SQLAlchemy URI. 

373 

374 Parameters 

375 ---------- 

376 uri : `str` 

377 A SQLAlchemy URI connection string. 

378 writeable : `bool`, optional 

379 If `True`, allow write operations on the database, including 

380 ``CREATE TABLE``. 

381 

382 Returns 

383 ------- 

384 engine : `sqlalchemy.engine.Engine` 

385 A database engine. 

386 

387 Notes 

388 ----- 

389 Subclasses that support other ways to connect to a database are 

390 encouraged to add optional arguments to their implementation of this 

391 method, as long as they maintain compatibility with the base class 

392 call signature. 

393 """ 

394 raise NotImplementedError() 

395 

396 @classmethod 

397 @abstractmethod 

398 def fromEngine(cls, engine: sqlalchemy.engine.Engine, *, origin: int, 

399 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

400 """Create a new `Database` from an existing `sqlalchemy.engine.Engine`. 

401 

402 Parameters 

403 ---------- 

404 engine : `sqllachemy.engine.Engine` 

405 The engine for the database. May be shared between `Database` 

406 instances. 

407 origin : `int` 

408 An integer ID that should be used as the default for any datasets, 

409 quanta, or other entities that use a (autoincrement, origin) 

410 compound primary key. 

411 namespace : `str`, optional 

412 A different database namespace (i.e. schema) the new instance 

413 should be associated with. If `None` (default), the namespace 

414 (if any) is inferred from the connection. 

415 writeable : `bool`, optional 

416 If `True`, allow write operations on the database, including 

417 ``CREATE TABLE``. 

418 

419 Returns 

420 ------- 

421 db : `Database` 

422 A new `Database` instance. 

423 

424 Notes 

425 ----- 

426 This method allows different `Database` instances to share the same 

427 engine, which is desirable when they represent different namespaces 

428 can be queried together. 

429 """ 

430 raise NotImplementedError() 

431 

432 @contextmanager 

433 def session(self) -> Iterator: 

434 """Return a context manager that represents a session (persistent 

435 connection to a database). 

436 """ 

437 if self._session_connection is not None: 

438 # session already started, just reuse that 

439 yield Session(self) 

440 else: 

441 # open new connection and close it when done 

442 self._session_connection = self._engine.connect() 

443 yield Session(self) 

444 self._session_connection.close() 

445 self._session_connection = None 

446 # Temporary tables only live within session 

447 self._tempTables = set() 

448 

449 @contextmanager 

450 def transaction(self, *, interrupting: bool = False, savepoint: bool = False, 

451 lock: Iterable[sqlalchemy.schema.Table] = ()) -> Iterator: 

452 """Return a context manager that represents a transaction. 

453 

454 Parameters 

455 ---------- 

456 interrupting : `bool`, optional 

457 If `True` (`False` is default), this transaction block may not be 

458 nested without an outer one, and attempting to do so is a logic 

459 (i.e. assertion) error. 

460 savepoint : `bool`, optional 

461 If `True` (`False` is default), create a `SAVEPOINT`, allowing 

462 exceptions raised by the database (e.g. due to constraint 

463 violations) during this transaction's context to be caught outside 

464 it without also rolling back all operations in an outer transaction 

465 block. If `False`, transactions may still be nested, but a 

466 rollback may be generated at any level and affects all levels, and 

467 commits are deferred until the outermost block completes. If any 

468 outer transaction block was created with ``savepoint=True``, all 

469 inner blocks will be as well (regardless of the actual value 

470 passed). This has no effect if this is the outermost transaction. 

471 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

472 A list of tables to lock for the duration of this transaction. 

473 These locks are guaranteed to prevent concurrent writes and allow 

474 this transaction (only) to acquire the same locks (others should 

475 block), but only prevent concurrent reads if the database engine 

476 requires that in order to block concurrent writes. 

477 

478 Notes 

479 ----- 

480 All transactions on a connection managed by one or more `Database` 

481 instances _must_ go through this method, or transaction state will not 

482 be correctly managed. 

483 """ 

484 # need a connection, use session to manage it 

485 with self.session(): 

486 assert self._session_connection is not None 

487 connection = self._session_connection 

488 assert not (interrupting and connection.in_transaction()), ( 

489 "Logic error in transaction nesting: an operation that would " 

490 "interrupt the active transaction context has been requested." 

491 ) 

492 # We remember whether we are already in a SAVEPOINT transaction via 

493 # the connection object's 'info' dict, which is explicitly for user 

494 # information like this. This is safer than a regular `Database` 

495 # instance attribute, because it guards against multiple `Database` 

496 # instances sharing the same connection. The need to use our own 

497 # flag here to track whether we're in a nested transaction should 

498 # go away in SQLAlchemy 1.4, which seems to have a 

499 # `Connection.in_nested_transaction()` method. 

500 savepoint = savepoint or connection.info.get(_IN_SAVEPOINT_TRANSACTION, False) 

501 connection.info[_IN_SAVEPOINT_TRANSACTION] = savepoint 

502 if connection.in_transaction() and savepoint: 

503 trans = connection.begin_nested() 

504 else: 

505 # Use a regular (non-savepoint) transaction always for the 

506 # outermost context, as well as when a savepoint was not 

507 # requested. 

508 trans = connection.begin() 

509 self._lockTables(lock) 

510 try: 

511 yield 

512 trans.commit() 

513 except BaseException: 

514 trans.rollback() 

515 raise 

516 finally: 

517 if not connection.in_transaction(): 

518 connection.info.pop(_IN_SAVEPOINT_TRANSACTION, None) 

519 

520 @property 

521 def _connection(self) -> sqlalchemy.engine.Connectable: 

522 """Object that can be used to execute queries 

523 (`sqlalchemy.engine.Connectable`) 

524 """ 

525 return self._session_connection or self._engine 

526 

527 @abstractmethod 

528 def _lockTables(self, tables: Iterable[sqlalchemy.schema.Table] = ()) -> None: 

529 """Acquire locks on the given tables. 

530 

531 This is an implementation hook for subclasses, called by `transaction`. 

532 It should not be called directly by other code. 

533 

534 Parameters 

535 ---------- 

536 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

537 A list of tables to lock for the duration of this transaction. 

538 These locks are guaranteed to prevent concurrent writes and allow 

539 this transaction (only) to acquire the same locks (others should 

540 block), but only prevent concurrent reads if the database engine 

541 requires that in order to block concurrent writes. 

542 """ 

543 raise NotImplementedError() 

544 

545 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool: 

546 """Check whether a table is writeable, either because the database 

547 connection is read-write or the table is a temporary table. 

548 

549 Parameters 

550 ---------- 

551 table : `sqlalchemy.schema.Table` 

552 SQLAlchemy table object to check. 

553 

554 Returns 

555 ------- 

556 writeable : `bool` 

557 Whether this table is writeable. 

558 """ 

559 return self.isWriteable() or table.key in self._tempTables 

560 

561 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None: 

562 """Raise if the given table is not writeable, either because the 

563 database connection is read-write or the table is a temporary table. 

564 

565 Parameters 

566 ---------- 

567 table : `sqlalchemy.schema.Table` 

568 SQLAlchemy table object to check. 

569 msg : `str`, optional 

570 If provided, raise `ReadOnlyDatabaseError` instead of returning 

571 `False`, with this message. 

572 """ 

573 if not self.isTableWriteable(table): 

574 raise ReadOnlyDatabaseError(msg) 

575 

576 @contextmanager 

577 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]: 

578 """Return a context manager in which the database's static DDL schema 

579 can be declared. 

580 

581 Parameters 

582 ---------- 

583 create : `bool` 

584 If `True`, attempt to create all tables at the end of the context. 

585 If `False`, they will be assumed to already exist. 

586 

587 Returns 

588 ------- 

589 schema : `StaticTablesContext` 

590 A helper object that is used to add new tables. 

591 

592 Raises 

593 ------ 

594 ReadOnlyDatabaseError 

595 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

596 and one or more declared tables do not already exist. 

597 

598 Examples 

599 -------- 

600 Given a `Database` instance ``db``:: 

601 

602 with db.declareStaticTables(create=True) as schema: 

603 schema.addTable("table1", TableSpec(...)) 

604 schema.addTable("table2", TableSpec(...)) 

605 

606 Notes 

607 ----- 

608 A database's static DDL schema must be declared before any dynamic 

609 tables are managed via calls to `ensureTableExists` or 

610 `getExistingTable`. The order in which static schema tables are added 

611 inside the context block is unimportant; they will automatically be 

612 sorted and added in an order consistent with their foreign key 

613 relationships. 

614 """ 

615 if create and not self.isWriteable(): 

616 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

617 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

618 try: 

619 context = StaticTablesContext(self) 

620 if create and context._tableNames: 

621 # Looks like database is already initalized, to avoid danger 

622 # of modifying/destroying valid schema we refuse to do 

623 # anything in this case 

624 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.") 

625 yield context 

626 for table, foreignKey in context._foreignKeys: 

627 table.append_constraint(foreignKey) 

628 if create: 

629 if self.namespace is not None: 

630 if self.namespace not in context._inspector.get_schema_names(): 

631 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

632 # In our tables we have columns that make use of sqlalchemy 

633 # Sequence objects. There is currently a bug in sqlalchemy that 

634 # causes a deprecation warning to be thrown on a property of 

635 # the Sequence object when the repr for the sequence is 

636 # created. Here a filter is used to catch these deprecation 

637 # warnings when tables are created. 

638 with warnings.catch_warnings(): 

639 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

640 self._metadata.create_all(self._connection) 

641 # call all initializer methods sequentially 

642 for init in context._initializers: 

643 init(self) 

644 except BaseException: 

645 self._metadata = None 

646 raise 

647 

648 @abstractmethod 

649 def isWriteable(self) -> bool: 

650 """Return `True` if this database can be modified by this client. 

651 """ 

652 raise NotImplementedError() 

653 

654 @abstractmethod 

655 def __str__(self) -> str: 

656 """Return a human-readable identifier for this `Database`, including 

657 any namespace or schema that identifies its names within a `Registry`. 

658 """ 

659 raise NotImplementedError() 

660 

661 @property 

662 def dialect(self) -> sqlalchemy.engine.Dialect: 

663 """The SQLAlchemy dialect for this database engine 

664 (`sqlalchemy.engine.Dialect`). 

665 """ 

666 return self._engine.dialect 

667 

668 def shrinkDatabaseEntityName(self, original: str) -> str: 

669 """Return a version of the given name that fits within this database 

670 engine's length limits for table, constraint, indexes, and sequence 

671 names. 

672 

673 Implementations should not assume that simple truncation is safe, 

674 because multiple long names often begin with the same prefix. 

675 

676 The default implementation simply returns the given name. 

677 

678 Parameters 

679 ---------- 

680 original : `str` 

681 The original name. 

682 

683 Returns 

684 ------- 

685 shrunk : `str` 

686 The new, possibly shortened name. 

687 """ 

688 return original 

689 

690 def expandDatabaseEntityName(self, shrunk: str) -> str: 

691 """Retrieve the original name for a database entity that was too long 

692 to fit within the database engine's limits. 

693 

694 Parameters 

695 ---------- 

696 original : `str` 

697 The original name. 

698 

699 Returns 

700 ------- 

701 shrunk : `str` 

702 The new, possibly shortened name. 

703 """ 

704 return shrunk 

705 

706 def _mangleTableName(self, name: str) -> str: 

707 """Map a logical, user-visible table name to the true table name used 

708 in the database. 

709 

710 The default implementation returns the given name unchanged. 

711 

712 Parameters 

713 ---------- 

714 name : `str` 

715 Input table name. Should not include a namespace (i.e. schema) 

716 prefix. 

717 

718 Returns 

719 ------- 

720 mangled : `str` 

721 Mangled version of the table name (still with no namespace prefix). 

722 

723 Notes 

724 ----- 

725 Reimplementations of this method must be idempotent - mangling an 

726 already-mangled name must have no effect. 

727 """ 

728 return name 

729 

730 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]: 

731 """Create constraints based on this spec. 

732 

733 Parameters 

734 ---------- 

735 table : `str` 

736 Name of the table this column is being added to. 

737 spec : `FieldSpec` 

738 Specification for the field to be added. 

739 

740 Returns 

741 ------- 

742 constraint : `list` of `sqlalchemy.CheckConstraint` 

743 Constraint added for this column. 

744 """ 

745 # By default we return no additional constraints 

746 return [] 

747 

748 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

749 **kwargs: Any) -> sqlalchemy.schema.Column: 

750 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

751 

752 Parameters 

753 ---------- 

754 table : `str` 

755 Name of the table this column is being added to. 

756 spec : `FieldSpec` 

757 Specification for the field to be added. 

758 metadata : `sqlalchemy.MetaData` 

759 SQLAlchemy representation of the DDL schema this field's table is 

760 being added to. 

761 **kwargs 

762 Additional keyword arguments to forward to the 

763 `sqlalchemy.schema.Column` constructor. This is provided to make 

764 it easier for derived classes to delegate to ``super()`` while 

765 making only minor changes. 

766 

767 Returns 

768 ------- 

769 column : `sqlalchemy.schema.Column` 

770 SQLAlchemy representation of the field. 

771 """ 

772 args = [spec.name, spec.getSizedColumnType()] 

773 if spec.autoincrement: 

774 # Generate a sequence to use for auto incrementing for databases 

775 # that do not support it natively. This will be ignored by 

776 # sqlalchemy for databases that do support it. 

777 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

778 metadata=metadata)) 

779 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

780 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

781 comment=spec.doc, server_default=spec.default, **kwargs) 

782 

783 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

784 **kwargs: Any) -> sqlalchemy.schema.ForeignKeyConstraint: 

785 """Convert a `ForeignKeySpec` to a 

786 `sqlalchemy.schema.ForeignKeyConstraint`. 

787 

788 Parameters 

789 ---------- 

790 table : `str` 

791 Name of the table this foreign key is being added to. 

792 spec : `ForeignKeySpec` 

793 Specification for the foreign key to be added. 

794 metadata : `sqlalchemy.MetaData` 

795 SQLAlchemy representation of the DDL schema this constraint is 

796 being added to. 

797 **kwargs 

798 Additional keyword arguments to forward to the 

799 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

800 provided to make it easier for derived classes to delegate to 

801 ``super()`` while making only minor changes. 

802 

803 Returns 

804 ------- 

805 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

806 SQLAlchemy representation of the constraint. 

807 """ 

808 name = self.shrinkDatabaseEntityName( 

809 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

810 + list(spec.target) + list(spec.source)) 

811 ) 

812 return sqlalchemy.schema.ForeignKeyConstraint( 

813 spec.source, 

814 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

815 name=name, 

816 ondelete=spec.onDelete 

817 ) 

818 

819 def _convertExclusionConstraintSpec(self, table: str, 

820 spec: Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...], 

821 metadata: sqlalchemy.MetaData) -> sqlalchemy.schema.Constraint: 

822 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy 

823 constraint representation. 

824 

825 Parameters 

826 ---------- 

827 table : `str` 

828 Name of the table this constraint is being added to. 

829 spec : `tuple` [ `str` or `type` ] 

830 A tuple of `str` column names and the `type` object returned by 

831 `getTimespanRepresentation` (which must appear exactly once), 

832 indicating the order of the columns in the index used to back the 

833 constraint. 

834 metadata : `sqlalchemy.MetaData` 

835 SQLAlchemy representation of the DDL schema this constraint is 

836 being added to. 

837 

838 Returns 

839 ------- 

840 constraint : `sqlalchemy.schema.Constraint` 

841 SQLAlchemy representation of the constraint. 

842 

843 Raises 

844 ------ 

845 NotImplementedError 

846 Raised if this database does not support exclusion constraints. 

847 """ 

848 raise NotImplementedError(f"Database {self} does not support exclusion constraints.") 

849 

850 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

851 **kwargs: Any) -> sqlalchemy.schema.Table: 

852 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

853 

854 Parameters 

855 ---------- 

856 spec : `TableSpec` 

857 Specification for the foreign key to be added. 

858 metadata : `sqlalchemy.MetaData` 

859 SQLAlchemy representation of the DDL schema this table is being 

860 added to. 

861 **kwargs 

862 Additional keyword arguments to forward to the 

863 `sqlalchemy.schema.Table` constructor. This is provided to make it 

864 easier for derived classes to delegate to ``super()`` while making 

865 only minor changes. 

866 

867 Returns 

868 ------- 

869 table : `sqlalchemy.schema.Table` 

870 SQLAlchemy representation of the table. 

871 

872 Notes 

873 ----- 

874 This method does not handle ``spec.foreignKeys`` at all, in order to 

875 avoid circular dependencies. These are added by higher-level logic in 

876 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

877 """ 

878 name = self._mangleTableName(name) 

879 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

880 

881 # Add any column constraints 

882 for fieldSpec in spec.fields: 

883 args.extend(self._makeColumnConstraints(name, fieldSpec)) 

884 

885 # Track indexes added for primary key and unique constraints, to make 

886 # sure we don't add duplicate explicit or foreign key indexes for 

887 # those. 

888 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)} 

889 args.extend( 

890 sqlalchemy.schema.UniqueConstraint( 

891 *columns, 

892 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

893 ) 

894 for columns in spec.unique 

895 ) 

896 allIndexes.update(spec.unique) 

897 args.extend( 

898 sqlalchemy.schema.Index( 

899 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

900 *columns, 

901 unique=(columns in spec.unique) 

902 ) 

903 for columns in spec.indexes if columns not in allIndexes 

904 ) 

905 allIndexes.update(spec.indexes) 

906 args.extend( 

907 sqlalchemy.schema.Index( 

908 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)), 

909 *fk.source, 

910 ) 

911 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes 

912 ) 

913 

914 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion) 

915 

916 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

917 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwargs) 

918 

919 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

920 """Ensure that a table with the given name and specification exists, 

921 creating it if necessary. 

922 

923 Parameters 

924 ---------- 

925 name : `str` 

926 Name of the table (not including namespace qualifiers). 

927 spec : `TableSpec` 

928 Specification for the table. This will be used when creating the 

929 table, and *may* be used when obtaining an existing table to check 

930 for consistency, but no such check is guaranteed. 

931 

932 Returns 

933 ------- 

934 table : `sqlalchemy.schema.Table` 

935 SQLAlchemy representation of the table. 

936 

937 Raises 

938 ------ 

939 ReadOnlyDatabaseError 

940 Raised if `isWriteable` returns `False`, and the table does not 

941 already exist. 

942 DatabaseConflictError 

943 Raised if the table exists but ``spec`` is inconsistent with its 

944 definition. 

945 

946 Notes 

947 ----- 

948 This method may not be called within transactions. It may be called on 

949 read-only databases if and only if the table does in fact already 

950 exist. 

951 

952 Subclasses may override this method, but usually should not need to. 

953 """ 

954 # TODO: if _engine is used to make a table then it uses separate 

955 # connection and should not interfere with current transaction 

956 assert self._session_connection is None or not self._session_connection.in_transaction(), \ 

957 "Table creation interrupts transactions." 

958 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

959 table = self.getExistingTable(name, spec) 

960 if table is not None: 

961 return table 

962 if not self.isWriteable(): 

963 raise ReadOnlyDatabaseError( 

964 f"Table {name} does not exist, and cannot be created " 

965 f"because database {self} is read-only." 

966 ) 

967 table = self._convertTableSpec(name, spec, self._metadata) 

968 for foreignKeySpec in spec.foreignKeys: 

969 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

970 table.create(self._connection) 

971 return table 

972 

973 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

974 """Obtain an existing table with the given name and specification. 

975 

976 Parameters 

977 ---------- 

978 name : `str` 

979 Name of the table (not including namespace qualifiers). 

980 spec : `TableSpec` 

981 Specification for the table. This will be used when creating the 

982 SQLAlchemy representation of the table, and it is used to 

983 check that the actual table in the database is consistent. 

984 

985 Returns 

986 ------- 

987 table : `sqlalchemy.schema.Table` or `None` 

988 SQLAlchemy representation of the table, or `None` if it does not 

989 exist. 

990 

991 Raises 

992 ------ 

993 DatabaseConflictError 

994 Raised if the table exists but ``spec`` is inconsistent with its 

995 definition. 

996 

997 Notes 

998 ----- 

999 This method can be called within transactions and never modifies the 

1000 database. 

1001 

1002 Subclasses may override this method, but usually should not need to. 

1003 """ 

1004 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

1005 name = self._mangleTableName(name) 

1006 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

1007 if table is not None: 

1008 if spec.fields.names != set(table.columns.keys()): 

1009 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

1010 f"specification has columns {list(spec.fields.names)}, while " 

1011 f"the previous definition has {list(table.columns.keys())}.") 

1012 else: 

1013 inspector = sqlalchemy.inspect(self._connection) 

1014 if name in inspector.get_table_names(schema=self.namespace): 

1015 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

1016 table = self._convertTableSpec(name, spec, self._metadata) 

1017 for foreignKeySpec in spec.foreignKeys: 

1018 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

1019 return table 

1020 return table 

1021 

1022 @classmethod 

1023 def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]: 

1024 """Return a `type` that encapsulates the way `Timespan` objects are 

1025 stored in this database. 

1026 

1027 `Database` does not automatically use the return type of this method 

1028 anywhere else; calling code is responsible for making sure that DDL 

1029 and queries are consistent with it. 

1030 

1031 Returns 

1032 ------- 

1033 TimespanReprClass : `type` (`TimespanDatabaseRepresention` subclass) 

1034 A type that encapsulates the way `Timespan` objects should be 

1035 stored in this database. 

1036 

1037 Notes 

1038 ----- 

1039 There are two big reasons we've decided to keep timespan-mangling logic 

1040 outside the `Database` implementations, even though the choice of 

1041 representation is ultimately up to a `Database` implementation: 

1042 

1043 - Timespans appear in relatively few tables and queries in our 

1044 typical usage, and the code that operates on them is already aware 

1045 that it is working with timespans. In contrast, a 

1046 timespan-representation-aware implementation of, say, `insert`, 

1047 would need to have extra logic to identify when timespan-mangling 

1048 needed to occur, which would usually be useless overhead. 

1049 

1050 - SQLAlchemy's rich SELECT query expression system has no way to wrap 

1051 multiple columns in a single expression object (the ORM does, but 

1052 we are not using the ORM). So we would have to wrap _much_ more of 

1053 that code in our own interfaces to encapsulate timespan 

1054 representations there. 

1055 """ 

1056 return TimespanDatabaseRepresentation.Compound 

1057 

1058 @classmethod 

1059 def getSpatialRegionRepresentation(cls) -> Type[SpatialRegionDatabaseRepresentation]: 

1060 """Return a `type` that encapsulates the way `lsst.sphgeom.Region` 

1061 objects are stored in this database. 

1062 

1063 `Database` does not automatically use the return type of this method 

1064 anywhere else; calling code is responsible for making sure that DDL 

1065 and queries are consistent with it. 

1066 

1067 Returns 

1068 ------- 

1069 RegionReprClass : `type` (`SpatialRegionDatabaseRepresention` subclass) 

1070 A type that encapsulates the way `lsst.sphgeom.Region` objects 

1071 should be stored in this database. 

1072 

1073 Notes 

1074 ----- 

1075 See `getTimespanRepresentation` for comments on why this method is not 

1076 more tightly integrated with the rest of the `Database` interface. 

1077 """ 

1078 return SpatialRegionDatabaseRepresentation 

1079 

1080 def sync(self, table: sqlalchemy.schema.Table, *, 

1081 keys: Dict[str, Any], 

1082 compared: Optional[Dict[str, Any]] = None, 

1083 extra: Optional[Dict[str, Any]] = None, 

1084 returning: Optional[Sequence[str]] = None, 

1085 update: bool = False, 

1086 ) -> Tuple[Optional[Dict[str, Any]], Union[bool, Dict[str, Any]]]: 

1087 """Insert into a table as necessary to ensure database contains 

1088 values equivalent to the given ones. 

1089 

1090 Parameters 

1091 ---------- 

1092 table : `sqlalchemy.schema.Table` 

1093 Table to be queried and possibly inserted into. 

1094 keys : `dict` 

1095 Column name-value pairs used to search for an existing row; must 

1096 be a combination that can be used to select a single row if one 

1097 exists. If such a row does not exist, these values are used in 

1098 the insert. 

1099 compared : `dict`, optional 

1100 Column name-value pairs that are compared to those in any existing 

1101 row. If such a row does not exist, these rows are used in the 

1102 insert. 

1103 extra : `dict`, optional 

1104 Column name-value pairs that are ignored if a matching row exists, 

1105 but used in an insert if one is necessary. 

1106 returning : `~collections.abc.Sequence` of `str`, optional 

1107 The names of columns whose values should be returned. 

1108 update : `bool`, optional 

1109 If `True` (`False` is default), update the existing row with the 

1110 values in ``compared`` instead of raising `DatabaseConflictError`. 

1111 

1112 Returns 

1113 ------- 

1114 row : `dict`, optional 

1115 The value of the fields indicated by ``returning``, or `None` if 

1116 ``returning`` is `None`. 

1117 inserted_or_updated : `bool` or `dict` 

1118 If `True`, a new row was inserted; if `False`, a matching row 

1119 already existed. If a `dict` (only possible if ``update=True``), 

1120 then an existing row was updated, and the dict maps the names of 

1121 the updated columns to their *old* values (new values can be 

1122 obtained from ``compared``). 

1123 

1124 Raises 

1125 ------ 

1126 DatabaseConflictError 

1127 Raised if the values in ``compared`` do not match the values in the 

1128 database. 

1129 ReadOnlyDatabaseError 

1130 Raised if `isWriteable` returns `False`, and no matching record 

1131 already exists. 

1132 

1133 Notes 

1134 ----- 

1135 May be used inside transaction contexts, so implementations may not 

1136 perform operations that interrupt transactions. 

1137 

1138 It may be called on read-only databases if and only if the matching row 

1139 does in fact already exist. 

1140 """ 

1141 

1142 def check() -> Tuple[int, Optional[Dict[str, Any]], Optional[List]]: 

1143 """Query for a row that matches the ``key`` argument, and compare 

1144 to what was given by the caller. 

1145 

1146 Returns 

1147 ------- 

1148 n : `int` 

1149 Number of matching rows. ``n != 1`` is always an error, but 

1150 it's a different kind of error depending on where `check` is 

1151 being called. 

1152 bad : `dict` or `None` 

1153 The subset of the keys of ``compared`` for which the existing 

1154 values did not match the given one, mapped to the existing 

1155 values in the database. Once again, ``not bad`` is always an 

1156 error, but a different kind on context. `None` if ``n != 1`` 

1157 result : `list` or `None` 

1158 Results in the database that correspond to the columns given 

1159 in ``returning``, or `None` if ``returning is None``. 

1160 """ 

1161 toSelect: Set[str] = set() 

1162 if compared is not None: 

1163 toSelect.update(compared.keys()) 

1164 if returning is not None: 

1165 toSelect.update(returning) 

1166 if not toSelect: 

1167 # Need to select some column, even if we just want to see 

1168 # how many rows we get back. 

1169 toSelect.add(next(iter(keys.keys()))) 

1170 selectSql = sqlalchemy.sql.select( 

1171 [table.columns[k].label(k) for k in toSelect] 

1172 ).select_from(table).where( 

1173 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

1174 ) 

1175 fetched = list(self._connection.execute(selectSql).fetchall()) 

1176 if len(fetched) != 1: 

1177 return len(fetched), None, None 

1178 existing = fetched[0] 

1179 if compared is not None: 

1180 

1181 def safeNotEqual(a: Any, b: Any) -> bool: 

1182 if isinstance(a, astropy.time.Time): 

1183 return not time_utils.TimeConverter().times_equal(a, b) 

1184 return a != b 

1185 

1186 inconsistencies = { 

1187 k: existing[k] 

1188 for k, v in compared.items() 

1189 if safeNotEqual(existing[k], v) 

1190 } 

1191 else: 

1192 inconsistencies = {} 

1193 if returning is not None: 

1194 toReturn: Optional[list] = [existing[k] for k in returning] 

1195 else: 

1196 toReturn = None 

1197 return 1, inconsistencies, toReturn 

1198 

1199 def format_bad(inconsistencies: Dict[str, Any]) -> str: 

1200 """Format the 'bad' dictionary of existing values returned by 

1201 ``check`` into a string suitable for an error message. 

1202 """ 

1203 assert compared is not None, "Should not be able to get inconsistencies without comparing." 

1204 return ", ".join(f"{k}: {v!r} != {compared[k]!r}" for k, v in inconsistencies.items()) 

1205 

1206 if self.isTableWriteable(table): 

1207 # Try an insert first, but allow it to fail (in only specific 

1208 # ways). 

1209 row = keys.copy() 

1210 if compared is not None: 

1211 row.update(compared) 

1212 if extra is not None: 

1213 row.update(extra) 

1214 with self.transaction(lock=[table]): 

1215 inserted = bool(self.ensure(table, row)) 

1216 inserted_or_updated: Union[bool, Dict[str, Any]] 

1217 # Need to perform check() for this branch inside the 

1218 # transaction, so we roll back an insert that didn't do 

1219 # what we expected. That limits the extent to which we 

1220 # can reduce duplication between this block and the other 

1221 # ones that perform similar logic. 

1222 n, bad, result = check() 

1223 if n < 1: 

1224 raise ConflictingDefinitionError( 

1225 f"Attempted to ensure {row} exists by inserting it with ON CONFLICT IGNORE, " 

1226 f"but a post-insert query on {keys} returned no results. " 

1227 f"Insert was {'' if inserted else 'not '}reported as successful. " 

1228 "This can occur if the insert violated a database constraint other than the " 

1229 "unique constraint or primary key used to identify the row in this call." 

1230 ) 

1231 elif n > 1: 

1232 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

1233 f"unique constraint for table {table.name}.") 

1234 elif bad: 

1235 assert compared is not None, \ 

1236 "Should not be able to get inconsistencies without comparing." 

1237 if inserted: 

1238 raise RuntimeError( 

1239 f"Conflict ({bad}) in sync after successful insert; this is " 

1240 "possible if the same table is being updated by a concurrent " 

1241 "process that isn't using sync, but it may also be a bug in " 

1242 "daf_butler." 

1243 ) 

1244 elif update: 

1245 self._connection.execute( 

1246 table.update().where( 

1247 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

1248 ).values( 

1249 **{k: compared[k] for k in bad.keys()} 

1250 ) 

1251 ) 

1252 inserted_or_updated = bad 

1253 else: 

1254 raise DatabaseConflictError( 

1255 f"Conflict in sync for table {table.name} on column(s) {format_bad(bad)}." 

1256 ) 

1257 else: 

1258 inserted_or_updated = inserted 

1259 else: 

1260 # Database is not writeable; just see if the row exists. 

1261 n, bad, result = check() 

1262 if n < 1: 

1263 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

1264 elif n > 1: 

1265 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

1266 elif bad: 

1267 if update: 

1268 raise ReadOnlyDatabaseError("sync needs to update, but database is read-only.") 

1269 else: 

1270 raise DatabaseConflictError( 

1271 f"Conflict in sync for table {table.name} on column(s) {format_bad(bad)}." 

1272 ) 

1273 inserted_or_updated = False 

1274 if returning is None: 

1275 return None, inserted_or_updated 

1276 else: 

1277 assert result is not None 

1278 return {k: v for k, v in zip(returning, result)}, inserted_or_updated 

1279 

1280 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

1281 select: Optional[sqlalchemy.sql.Select] = None, 

1282 names: Optional[Iterable[str]] = None, 

1283 ) -> Optional[List[int]]: 

1284 """Insert one or more rows into a table, optionally returning 

1285 autoincrement primary key values. 

1286 

1287 Parameters 

1288 ---------- 

1289 table : `sqlalchemy.schema.Table` 

1290 Table rows should be inserted into. 

1291 returnIds: `bool` 

1292 If `True` (`False` is default), return the values of the table's 

1293 autoincrement primary key field (which much exist). 

1294 select : `sqlalchemy.sql.Select`, optional 

1295 A SELECT query expression to insert rows from. Cannot be provided 

1296 with either ``rows`` or ``returnIds=True``. 

1297 names : `Iterable` [ `str` ], optional 

1298 Names of columns in ``table`` to be populated, ordered to match the 

1299 columns returned by ``select``. Ignored if ``select`` is `None`. 

1300 If not provided, the columns returned by ``select`` must be named 

1301 to match the desired columns of ``table``. 

1302 *rows 

1303 Positional arguments are the rows to be inserted, as dictionaries 

1304 mapping column name to value. The keys in all dictionaries must 

1305 be the same. 

1306 

1307 Returns 

1308 ------- 

1309 ids : `None`, or `list` of `int` 

1310 If ``returnIds`` is `True`, a `list` containing the inserted 

1311 values for the table's autoincrement primary key. 

1312 

1313 Raises 

1314 ------ 

1315 ReadOnlyDatabaseError 

1316 Raised if `isWriteable` returns `False` when this method is called. 

1317 

1318 Notes 

1319 ----- 

1320 The default implementation uses bulk insert syntax when ``returnIds`` 

1321 is `False`, and a loop over single-row insert operations when it is 

1322 `True`. 

1323 

1324 Derived classes should reimplement when they can provide a more 

1325 efficient implementation (especially for the latter case). 

1326 

1327 May be used inside transaction contexts, so implementations may not 

1328 perform operations that interrupt transactions. 

1329 """ 

1330 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.") 

1331 if select is not None and (rows or returnIds): 

1332 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.") 

1333 if not rows and select is None: 

1334 if returnIds: 

1335 return [] 

1336 else: 

1337 return None 

1338 if not returnIds: 

1339 if select is not None: 

1340 if names is None: 

1341 # columns() is deprecated since 1.4, but 

1342 # selected_columns() method did not exist in 1.3. 

1343 if hasattr(select, "selected_columns"): 

1344 names = select.selected_columns.keys() 

1345 else: 

1346 names = select.columns.keys() 

1347 self._connection.execute(table.insert().from_select(names, select)) 

1348 else: 

1349 self._connection.execute(table.insert(), *rows) 

1350 return None 

1351 else: 

1352 sql = table.insert() 

1353 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

1354 

1355 @abstractmethod 

1356 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None: 

1357 """Insert one or more rows into a table, replacing any existing rows 

1358 for which insertion of a new row would violate the primary key 

1359 constraint. 

1360 

1361 Parameters 

1362 ---------- 

1363 table : `sqlalchemy.schema.Table` 

1364 Table rows should be inserted into. 

1365 *rows 

1366 Positional arguments are the rows to be inserted, as dictionaries 

1367 mapping column name to value. The keys in all dictionaries must 

1368 be the same. 

1369 

1370 Raises 

1371 ------ 

1372 ReadOnlyDatabaseError 

1373 Raised if `isWriteable` returns `False` when this method is called. 

1374 

1375 Notes 

1376 ----- 

1377 May be used inside transaction contexts, so implementations may not 

1378 perform operations that interrupt transactions. 

1379 

1380 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

1381 exception when a constraint other than the primary key would be 

1382 violated. 

1383 

1384 Implementations are not required to support `replace` on tables 

1385 with autoincrement keys. 

1386 """ 

1387 raise NotImplementedError() 

1388 

1389 @abstractmethod 

1390 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict) -> int: 

1391 """Insert one or more rows into a table, skipping any rows for which 

1392 insertion would violate any constraint. 

1393 

1394 Parameters 

1395 ---------- 

1396 table : `sqlalchemy.schema.Table` 

1397 Table rows should be inserted into. 

1398 *rows 

1399 Positional arguments are the rows to be inserted, as dictionaries 

1400 mapping column name to value. The keys in all dictionaries must 

1401 be the same. 

1402 

1403 Returns 

1404 ------- 

1405 count : `int` 

1406 The number of rows actually inserted. 

1407 

1408 Raises 

1409 ------ 

1410 ReadOnlyDatabaseError 

1411 Raised if `isWriteable` returns `False` when this method is called. 

1412 This is raised even if the operation would do nothing even on a 

1413 writeable database. 

1414 

1415 Notes 

1416 ----- 

1417 May be used inside transaction contexts, so implementations may not 

1418 perform operations that interrupt transactions. 

1419 

1420 Implementations are not required to support `ensure` on tables 

1421 with autoincrement keys. 

1422 """ 

1423 raise NotImplementedError() 

1424 

1425 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

1426 """Delete one or more rows from a table. 

1427 

1428 Parameters 

1429 ---------- 

1430 table : `sqlalchemy.schema.Table` 

1431 Table that rows should be deleted from. 

1432 columns: `~collections.abc.Iterable` of `str` 

1433 The names of columns that will be used to constrain the rows to 

1434 be deleted; these will be combined via ``AND`` to form the 

1435 ``WHERE`` clause of the delete query. 

1436 *rows 

1437 Positional arguments are the keys of rows to be deleted, as 

1438 dictionaries mapping column name to value. The keys in all 

1439 dictionaries must be exactly the names in ``columns``. 

1440 

1441 Returns 

1442 ------- 

1443 count : `int` 

1444 Number of rows deleted. 

1445 

1446 Raises 

1447 ------ 

1448 ReadOnlyDatabaseError 

1449 Raised if `isWriteable` returns `False` when this method is called. 

1450 

1451 Notes 

1452 ----- 

1453 May be used inside transaction contexts, so implementations may not 

1454 perform operations that interrupt transactions. 

1455 

1456 The default implementation should be sufficient for most derived 

1457 classes. 

1458 """ 

1459 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.") 

1460 if columns and not rows: 

1461 # If there are no columns, this operation is supposed to delete 

1462 # everything (so we proceed as usual). But if there are columns, 

1463 # but no rows, it was a constrained bulk operation where the 

1464 # constraint is that no rows match, and we should short-circuit 

1465 # while reporting that no rows were affected. 

1466 return 0 

1467 sql = table.delete() 

1468 columns = list(columns) # Force iterators to list 

1469 

1470 # More efficient to use IN operator if there is only one 

1471 # variable changing across all rows. 

1472 content: Dict[str, Set] = defaultdict(set) 

1473 if len(columns) == 1: 

1474 # Nothing to calculate since we can always use IN 

1475 column = columns[0] 

1476 changing_columns = [column] 

1477 content[column] = set(row[column] for row in rows) 

1478 else: 

1479 for row in rows: 

1480 for k, v in row.items(): 

1481 content[k].add(v) 

1482 changing_columns = [col for col, values in content.items() if len(values) > 1] 

1483 

1484 if len(changing_columns) != 1: 

1485 # More than one column changes each time so do explicit bind 

1486 # parameters and have each row processed separately. 

1487 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

1488 if whereTerms: 

1489 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

1490 return self._connection.execute(sql, *rows).rowcount 

1491 else: 

1492 # One of the columns has changing values but any others are 

1493 # fixed. In this case we can use an IN operator and be more 

1494 # efficient. 

1495 name = changing_columns.pop() 

1496 

1497 # Simple where clause for the unchanging columns 

1498 clauses = [] 

1499 for k, v in content.items(): 

1500 if k == name: 

1501 continue 

1502 column = table.columns[k] 

1503 # The set only has one element 

1504 clauses.append(column == v.pop()) 

1505 

1506 # The IN operator will not work for "infinite" numbers of 

1507 # rows so must batch it up into distinct calls. 

1508 in_content = list(content[name]) 

1509 n_elements = len(in_content) 

1510 

1511 rowcount = 0 

1512 iposn = 0 

1513 n_per_loop = 1_000 # Controls how many items to put in IN clause 

1514 for iposn in range(0, n_elements, n_per_loop): 

1515 endpos = iposn + n_per_loop 

1516 in_clause = table.columns[name].in_(in_content[iposn:endpos]) 

1517 

1518 newsql = sql.where(sqlalchemy.sql.and_(*clauses, in_clause)) 

1519 rowcount += self._connection.execute(newsql).rowcount 

1520 return rowcount 

1521 

1522 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

1523 """Update one or more rows in a table. 

1524 

1525 Parameters 

1526 ---------- 

1527 table : `sqlalchemy.schema.Table` 

1528 Table containing the rows to be updated. 

1529 where : `dict` [`str`, `str`] 

1530 A mapping from the names of columns that will be used to search for 

1531 existing rows to the keys that will hold these values in the 

1532 ``rows`` dictionaries. Note that these may not be the same due to 

1533 SQLAlchemy limitations. 

1534 *rows 

1535 Positional arguments are the rows to be updated. The keys in all 

1536 dictionaries must be the same, and may correspond to either a 

1537 value in the ``where`` dictionary or the name of a column to be 

1538 updated. 

1539 

1540 Returns 

1541 ------- 

1542 count : `int` 

1543 Number of rows matched (regardless of whether the update actually 

1544 modified them). 

1545 

1546 Raises 

1547 ------ 

1548 ReadOnlyDatabaseError 

1549 Raised if `isWriteable` returns `False` when this method is called. 

1550 

1551 Notes 

1552 ----- 

1553 May be used inside transaction contexts, so implementations may not 

1554 perform operations that interrupt transactions. 

1555 

1556 The default implementation should be sufficient for most derived 

1557 classes. 

1558 """ 

1559 self.assertTableWriteable(table, f"Cannot update read-only table {table}.") 

1560 if not rows: 

1561 return 0 

1562 sql = table.update().where( 

1563 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1564 ) 

1565 return self._connection.execute(sql, *rows).rowcount 

1566 

1567 def query(self, sql: sqlalchemy.sql.FromClause, 

1568 *args: Any, **kwargs: Any) -> sqlalchemy.engine.ResultProxy: 

1569 """Run a SELECT query against the database. 

1570 

1571 Parameters 

1572 ---------- 

1573 sql : `sqlalchemy.sql.FromClause` 

1574 A SQLAlchemy representation of a ``SELECT`` query. 

1575 *args 

1576 Additional positional arguments are forwarded to 

1577 `sqlalchemy.engine.Connection.execute`. 

1578 **kwargs 

1579 Additional keyword arguments are forwarded to 

1580 `sqlalchemy.engine.Connection.execute`. 

1581 

1582 Returns 

1583 ------- 

1584 result : `sqlalchemy.engine.ResultProxy` 

1585 Query results. 

1586 

1587 Notes 

1588 ----- 

1589 The default implementation should be sufficient for most derived 

1590 classes. 

1591 """ 

1592 # TODO: should we guard against non-SELECT queries here? 

1593 return self._connection.execute(sql, *args, **kwargs) 

1594 

1595 origin: int 

1596 """An integer ID that should be used as the default for any datasets, 

1597 quanta, or other entities that use a (autoincrement, origin) compound 

1598 primary key (`int`). 

1599 """ 

1600 

1601 namespace: Optional[str] 

1602 """The schema or namespace this database instance is associated with 

1603 (`str` or `None`). 

1604 """