Coverage for python/lsst/daf/butler/registry/interfaces/_database.py: 14%

407 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-24 23:50 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "SchemaAlreadyDefinedError", 

28 "StaticTablesContext", 

29] 

30 

31import uuid 

32import warnings 

33from abc import ABC, abstractmethod 

34from collections import defaultdict 

35from contextlib import contextmanager 

36from typing import ( 

37 Any, 

38 Callable, 

39 Dict, 

40 Iterable, 

41 Iterator, 

42 List, 

43 Optional, 

44 Sequence, 

45 Set, 

46 Tuple, 

47 Type, 

48 Union, 

49 final, 

50) 

51 

52import astropy.time 

53import sqlalchemy 

54 

55from ...core import SpatialRegionDatabaseRepresentation, TimespanDatabaseRepresentation, ddl, time_utils 

56from .._exceptions import ConflictingDefinitionError 

57 

58 

59def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None: 

60 """Test that the definition of a table in a `ddl.TableSpec` and from 

61 database introspection are consistent. 

62 

63 Parameters 

64 ---------- 

65 name : `str` 

66 Name of the table (only used in error messages). 

67 spec : `ddl.TableSpec` 

68 Specification of the table. 

69 inspection : `dict` 

70 Dictionary returned by 

71 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

72 

73 Raises 

74 ------ 

75 DatabaseConflictError 

76 Raised if the definitions are inconsistent. 

77 """ 

78 columnNames = [c["name"] for c in inspection] 

79 if spec.fields.names != set(columnNames): 

80 raise DatabaseConflictError( 

81 f"Table '{name}' exists but is defined differently in the database; " 

82 f"specification has columns {list(spec.fields.names)}, while the " 

83 f"table in the database has {columnNames}." 

84 ) 

85 

86 

87class ReadOnlyDatabaseError(RuntimeError): 

88 """Exception raised when a write operation is called on a read-only 

89 `Database`. 

90 """ 

91 

92 

93class DatabaseConflictError(ConflictingDefinitionError): 

94 """Exception raised when database content (row values or schema entities) 

95 are inconsistent with what this client expects. 

96 """ 

97 

98 

99class SchemaAlreadyDefinedError(RuntimeError): 

100 """Exception raised when trying to initialize database schema when some 

101 tables already exist. 

102 """ 

103 

104 

105class StaticTablesContext: 

106 """Helper class used to declare the static schema for a registry layer 

107 in a database. 

108 

109 An instance of this class is returned by `Database.declareStaticTables`, 

110 which should be the only way it should be constructed. 

111 """ 

112 

113 def __init__(self, db: Database, connection: sqlalchemy.engine.Connection): 

114 self._db = db 

115 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = [] 

116 self._inspector = sqlalchemy.inspect(connection) 

117 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

118 self._initializers: List[Callable[[Database], None]] = [] 

119 

120 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

121 """Add a new table to the schema, returning its sqlalchemy 

122 representation. 

123 

124 The new table may not actually be created until the end of the 

125 context created by `Database.declareStaticTables`, allowing tables 

126 to be declared in any order even in the presence of foreign key 

127 relationships. 

128 """ 

129 name = self._db._mangleTableName(name) 

130 if name in self._tableNames: 

131 _checkExistingTableDefinition( 

132 name, spec, self._inspector.get_columns(name, schema=self._db.namespace) 

133 ) 

134 metadata = self._db._metadata 

135 assert metadata is not None, "Guaranteed by context manager that returns this object." 

136 table = self._db._convertTableSpec(name, spec, metadata) 

137 for foreignKeySpec in spec.foreignKeys: 

138 self._foreignKeys.append((table, self._db._convertForeignKeySpec(name, foreignKeySpec, metadata))) 

139 return table 

140 

141 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

142 """Add a named tuple of tables to the schema, returning their 

143 SQLAlchemy representations in a named tuple of the same type. 

144 

145 The new tables may not actually be created until the end of the 

146 context created by `Database.declareStaticTables`, allowing tables 

147 to be declared in any order even in the presence of foreign key 

148 relationships. 

149 

150 Notes 

151 ----- 

152 ``specs`` *must* be an instance of a type created by 

153 `collections.namedtuple`, not just regular tuple, and the returned 

154 object is guaranteed to be the same. Because `~collections.namedtuple` 

155 is just a factory for `type` objects, not an actual type itself, 

156 we cannot represent this with type annotations. 

157 """ 

158 return specs._make( # type: ignore 

159 self.addTable(name, spec) for name, spec in zip(specs._fields, specs) # type: ignore 

160 ) 

161 

162 def addInitializer(self, initializer: Callable[[Database], None]) -> None: 

163 """Add a method that does one-time initialization of a database. 

164 

165 Initialization can mean anything that changes state of a database 

166 and needs to be done exactly once after database schema was created. 

167 An example for that could be population of schema attributes. 

168 

169 Parameters 

170 ---------- 

171 initializer : callable 

172 Method of a single argument which is a `Database` instance. 

173 """ 

174 self._initializers.append(initializer) 

175 

176 

177class Database(ABC): 

178 """An abstract interface that represents a particular database engine's 

179 representation of a single schema/namespace/database. 

180 

181 Parameters 

182 ---------- 

183 origin : `int` 

184 An integer ID that should be used as the default for any datasets, 

185 quanta, or other entities that use a (autoincrement, origin) compound 

186 primary key. 

187 engine : `sqlalchemy.engine.Engine` 

188 The SQLAlchemy engine for this `Database`. 

189 namespace : `str`, optional 

190 Name of the schema or namespace this instance is associated with. 

191 This is passed as the ``schema`` argument when constructing a 

192 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

193 avoid confusion between "schema means namespace" and "schema means 

194 table definitions". 

195 

196 Notes 

197 ----- 

198 `Database` requires all write operations to go through its special named 

199 methods. Our write patterns are sufficiently simple that we don't really 

200 need the full flexibility of SQL insert/update/delete syntax, and we need 

201 non-standard (but common) functionality in these operations sufficiently 

202 often that it seems worthwhile to provide our own generic API. 

203 

204 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

205 their SQLAlchemy representation) to be run, as we expect these to require 

206 significantly more sophistication while still being limited to standard 

207 SQL. 

208 

209 `Database` itself has several underscore-prefixed attributes: 

210 

211 - ``_engine``: SQLAlchemy object representing its engine. 

212 - ``_connection``: method returning a context manager for 

213 `sqlalchemy.engine.Connection` object. 

214 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

215 the tables and other schema entities. 

216 

217 These are considered protected (derived classes may access them, but other 

218 code should not), and read-only, aside from executing SQL via 

219 ``_connection``. 

220 """ 

221 

222 def __init__(self, *, origin: int, engine: sqlalchemy.engine.Engine, namespace: Optional[str] = None): 

223 self.origin = origin 

224 self.namespace = namespace 

225 self._engine = engine 

226 self._session_connection: Optional[sqlalchemy.engine.Connection] = None 

227 self._metadata: Optional[sqlalchemy.schema.MetaData] = None 

228 self._temp_tables: Set[str] = set() 

229 

230 def __repr__(self) -> str: 

231 # Rather than try to reproduce all the parameters used to create 

232 # the object, instead report the more useful information of the 

233 # connection URL. 

234 if self._engine.url.password is not None: 

235 uri = str(self._engine.url.set(password="***")) 

236 else: 

237 uri = str(self._engine.url) 

238 if self.namespace: 

239 uri += f"#{self.namespace}" 

240 return f'{type(self).__name__}("{uri}")' 

241 

242 @classmethod 

243 def makeDefaultUri(cls, root: str) -> Optional[str]: 

244 """Create a default connection URI appropriate for the given root 

245 directory, or `None` if there can be no such default. 

246 """ 

247 return None 

248 

249 @classmethod 

250 def fromUri( 

251 cls, uri: str, *, origin: int, namespace: Optional[str] = None, writeable: bool = True 

252 ) -> Database: 

253 """Construct a database from a SQLAlchemy URI. 

254 

255 Parameters 

256 ---------- 

257 uri : `str` 

258 A SQLAlchemy URI connection string. 

259 origin : `int` 

260 An integer ID that should be used as the default for any datasets, 

261 quanta, or other entities that use a (autoincrement, origin) 

262 compound primary key. 

263 namespace : `str`, optional 

264 A database namespace (i.e. schema) the new instance should be 

265 associated with. If `None` (default), the namespace (if any) is 

266 inferred from the URI. 

267 writeable : `bool`, optional 

268 If `True`, allow write operations on the database, including 

269 ``CREATE TABLE``. 

270 

271 Returns 

272 ------- 

273 db : `Database` 

274 A new `Database` instance. 

275 """ 

276 return cls.fromEngine( 

277 cls.makeEngine(uri, writeable=writeable), origin=origin, namespace=namespace, writeable=writeable 

278 ) 

279 

280 @classmethod 

281 @abstractmethod 

282 def makeEngine(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Engine: 

283 """Create a `sqlalchemy.engine.Engine` from a SQLAlchemy URI. 

284 

285 Parameters 

286 ---------- 

287 uri : `str` 

288 A SQLAlchemy URI connection string. 

289 writeable : `bool`, optional 

290 If `True`, allow write operations on the database, including 

291 ``CREATE TABLE``. 

292 

293 Returns 

294 ------- 

295 engine : `sqlalchemy.engine.Engine` 

296 A database engine. 

297 

298 Notes 

299 ----- 

300 Subclasses that support other ways to connect to a database are 

301 encouraged to add optional arguments to their implementation of this 

302 method, as long as they maintain compatibility with the base class 

303 call signature. 

304 """ 

305 raise NotImplementedError() 

306 

307 @classmethod 

308 @abstractmethod 

309 def fromEngine( 

310 cls, 

311 engine: sqlalchemy.engine.Engine, 

312 *, 

313 origin: int, 

314 namespace: Optional[str] = None, 

315 writeable: bool = True, 

316 ) -> Database: 

317 """Create a new `Database` from an existing `sqlalchemy.engine.Engine`. 

318 

319 Parameters 

320 ---------- 

321 engine : `sqlalchemy.engine.Engine` 

322 The engine for the database. May be shared between `Database` 

323 instances. 

324 origin : `int` 

325 An integer ID that should be used as the default for any datasets, 

326 quanta, or other entities that use a (autoincrement, origin) 

327 compound primary key. 

328 namespace : `str`, optional 

329 A different database namespace (i.e. schema) the new instance 

330 should be associated with. If `None` (default), the namespace 

331 (if any) is inferred from the connection. 

332 writeable : `bool`, optional 

333 If `True`, allow write operations on the database, including 

334 ``CREATE TABLE``. 

335 

336 Returns 

337 ------- 

338 db : `Database` 

339 A new `Database` instance. 

340 

341 Notes 

342 ----- 

343 This method allows different `Database` instances to share the same 

344 engine, which is desirable when they represent different namespaces 

345 can be queried together. 

346 """ 

347 raise NotImplementedError() 

348 

349 @final 

350 @contextmanager 

351 def session(self) -> Iterator[None]: 

352 """Return a context manager that represents a session (persistent 

353 connection to a database). 

354 

355 Returns 

356 ------- 

357 context : `AbstractContextManager` [ `None` ] 

358 A context manager that does not return a value when entered. 

359 

360 Notes 

361 ----- 

362 This method should be used when a sequence of read-only SQL operations 

363 will be performed in rapid succession *without* a requirement that they 

364 yield consistent results in the presence of concurrent writes (or, more 

365 rarely, when conflicting concurrent writes are rare/impossible and the 

366 session will be open long enough that a transaction is inadvisable). 

367 """ 

368 with self._session(): 

369 yield 

370 

371 @final 

372 @contextmanager 

373 def transaction( 

374 self, 

375 *, 

376 interrupting: bool = False, 

377 savepoint: bool = False, 

378 lock: Iterable[sqlalchemy.schema.Table] = (), 

379 for_temp_tables: bool = False, 

380 ) -> Iterator[None]: 

381 """Return a context manager that represents a transaction. 

382 

383 Parameters 

384 ---------- 

385 interrupting : `bool`, optional 

386 If `True` (`False` is default), this transaction block may not be 

387 nested without an outer one, and attempting to do so is a logic 

388 (i.e. assertion) error. 

389 savepoint : `bool`, optional 

390 If `True` (`False` is default), create a `SAVEPOINT`, allowing 

391 exceptions raised by the database (e.g. due to constraint 

392 violations) during this transaction's context to be caught outside 

393 it without also rolling back all operations in an outer transaction 

394 block. If `False`, transactions may still be nested, but a 

395 rollback may be generated at any level and affects all levels, and 

396 commits are deferred until the outermost block completes. If any 

397 outer transaction block was created with ``savepoint=True``, all 

398 inner blocks will be as well (regardless of the actual value 

399 passed). This has no effect if this is the outermost transaction. 

400 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

401 A list of tables to lock for the duration of this transaction. 

402 These locks are guaranteed to prevent concurrent writes and allow 

403 this transaction (only) to acquire the same locks (others should 

404 block), but only prevent concurrent reads if the database engine 

405 requires that in order to block concurrent writes. 

406 for_temp_tables : `bool`, optional 

407 If `True`, this transaction may involve creating temporary tables. 

408 

409 Returns 

410 ------- 

411 context : `AbstractContextManager` [ `None` ] 

412 A context manager that commits the transaction when it is exited 

413 without error and rolls back the transactoin when it is exited via 

414 an exception. 

415 

416 Notes 

417 ----- 

418 All transactions on a connection managed by one or more `Database` 

419 instances _must_ go through this method, or transaction state will not 

420 be correctly managed. 

421 """ 

422 with self._transaction( 

423 interrupting=interrupting, savepoint=savepoint, lock=lock, for_temp_tables=for_temp_tables 

424 ): 

425 yield 

426 

427 @contextmanager 

428 def temporary_table( 

429 self, spec: ddl.TableSpec, name: Optional[str] = None 

430 ) -> Iterator[sqlalchemy.schema.Table]: 

431 """Return a context manager that creates and then drops a temporary 

432 table. 

433 

434 Parameters 

435 ---------- 

436 spec : `ddl.TableSpec` 

437 Specification for the columns. Unique and foreign key constraints 

438 may be ignored. 

439 name : `str`, optional 

440 If provided, the name of the SQL construct. If not provided, an 

441 opaque but unique identifier is generated. 

442 

443 Returns 

444 ------- 

445 context : `AbstractContextManager` [ `sqlalchemy.schema.Table` ] 

446 A context manager that returns a SQLAlchemy representation of the 

447 temporary table when entered. 

448 

449 Notes 

450 ----- 

451 Temporary tables may be created, dropped, and written to even in 

452 read-only databases - at least according to the Python-level 

453 protections in the `Database` classes. Server permissions may say 

454 otherwise, but in that case they probably need to be modified to 

455 support the full range of expected read-only butler behavior. 

456 """ 

457 with self._session() as connection: 

458 table = self._make_temporary_table(connection, spec=spec, name=name) 

459 self._temp_tables.add(table.key) 

460 try: 

461 yield table 

462 finally: 

463 table.drop(connection) 

464 self._temp_tables.remove(table.key) 

465 

466 @contextmanager 

467 def _session(self) -> Iterator[sqlalchemy.engine.Connection]: 

468 """Protected implementation for `session` that actually returns the 

469 connection. 

470 

471 This method is for internal `Database` calls that need the actual 

472 SQLAlchemy connection object. It should be overridden by subclasses 

473 instead of `session` itself. 

474 

475 Returns 

476 ------- 

477 context : `AbstractContextManager` [ `sqlalchemy.engine.Connection` ] 

478 A context manager that returns a SQLALchemy connection when 

479 entered. 

480 

481 """ 

482 if self._session_connection is not None: 

483 # session already started, just reuse that 

484 yield self._session_connection 

485 else: 

486 try: 

487 # open new connection and close it when done 

488 self._session_connection = self._engine.connect() 

489 yield self._session_connection 

490 finally: 

491 if self._session_connection is not None: 

492 self._session_connection.close() 

493 self._session_connection = None 

494 # Temporary tables only live within session 

495 self._temp_tables = set() 

496 

497 @contextmanager 

498 def _transaction( 

499 self, 

500 *, 

501 interrupting: bool = False, 

502 savepoint: bool = False, 

503 lock: Iterable[sqlalchemy.schema.Table] = (), 

504 for_temp_tables: bool = False, 

505 ) -> Iterator[tuple[bool, sqlalchemy.engine.Connection]]: 

506 """Protected implementation for `transaction` that actually returns the 

507 connection and whether this is a new outermost transaction. 

508 

509 This method is for internal `Database` calls that need the actual 

510 SQLAlchemy connection object. It should be overridden by subclasses 

511 instead of `transaction` itself. 

512 

513 Parameters 

514 ---------- 

515 interrupting : `bool`, optional 

516 If `True` (`False` is default), this transaction block may not be 

517 nested without an outer one, and attempting to do so is a logic 

518 (i.e. assertion) error. 

519 savepoint : `bool`, optional 

520 If `True` (`False` is default), create a `SAVEPOINT`, allowing 

521 exceptions raised by the database (e.g. due to constraint 

522 violations) during this transaction's context to be caught outside 

523 it without also rolling back all operations in an outer transaction 

524 block. If `False`, transactions may still be nested, but a 

525 rollback may be generated at any level and affects all levels, and 

526 commits are deferred until the outermost block completes. If any 

527 outer transaction block was created with ``savepoint=True``, all 

528 inner blocks will be as well (regardless of the actual value 

529 passed). This has no effect if this is the outermost transaction. 

530 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

531 A list of tables to lock for the duration of this transaction. 

532 These locks are guaranteed to prevent concurrent writes and allow 

533 this transaction (only) to acquire the same locks (others should 

534 block), but only prevent concurrent reads if the database engine 

535 requires that in order to block concurrent writes. 

536 for_temp_tables : `bool`, optional 

537 If `True`, this transaction may involve creating temporary tables. 

538 

539 Returns 

540 ------- 

541 context : `AbstractContextManager` [ `tuple` [ `bool`, 

542 `sqlalchemy.engine.Connection` ] ] 

543 A context manager that commits the transaction when it is exited 

544 without error and rolls back the transactoin when it is exited via 

545 an exception. When entered, it returns a tuple of: 

546 

547 - ``is_new`` (`bool`): whether this is a new (outermost) 

548 transaction; 

549 - ``connection`` (`sqlalchemy.engine.Connection`): the connection. 

550 """ 

551 with self._session() as connection: 

552 already_in_transaction = connection.in_transaction() 

553 assert not (interrupting and already_in_transaction), ( 

554 "Logic error in transaction nesting: an operation that would " 

555 "interrupt the active transaction context has been requested." 

556 ) 

557 savepoint = savepoint or connection.in_nested_transaction() 

558 trans: sqlalchemy.engine.Transaction | None 

559 if already_in_transaction: 

560 if savepoint: 

561 trans = connection.begin_nested() 

562 else: 

563 # Nested non-savepoint transactions don't do anything. 

564 trans = None 

565 else: 

566 # Use a regular (non-savepoint) transaction always for the 

567 # outermost context. 

568 trans = connection.begin() 

569 self._lockTables(connection, lock) 

570 try: 

571 yield not already_in_transaction, connection 

572 if trans is not None: 

573 trans.commit() 

574 except BaseException: 

575 if trans is not None: 

576 trans.rollback() 

577 raise 

578 

579 @abstractmethod 

580 def _lockTables( 

581 self, connection: sqlalchemy.engine.Connection, tables: Iterable[sqlalchemy.schema.Table] = () 

582 ) -> None: 

583 """Acquire locks on the given tables. 

584 

585 This is an implementation hook for subclasses, called by `transaction`. 

586 It should not be called directly by other code. 

587 

588 Parameters 

589 ---------- 

590 connection : `sqlalchemy.engine.Connection` 

591 Database connection object. It is guaranteed that transaction is 

592 already in a progress for this connection. 

593 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

594 A list of tables to lock for the duration of this transaction. 

595 These locks are guaranteed to prevent concurrent writes and allow 

596 this transaction (only) to acquire the same locks (others should 

597 block), but only prevent concurrent reads if the database engine 

598 requires that in order to block concurrent writes. 

599 """ 

600 raise NotImplementedError() 

601 

602 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool: 

603 """Check whether a table is writeable, either because the database 

604 connection is read-write or the table is a temporary table. 

605 

606 Parameters 

607 ---------- 

608 table : `sqlalchemy.schema.Table` 

609 SQLAlchemy table object to check. 

610 

611 Returns 

612 ------- 

613 writeable : `bool` 

614 Whether this table is writeable. 

615 """ 

616 return self.isWriteable() or table.key in self._temp_tables 

617 

618 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None: 

619 """Raise if the given table is not writeable, either because the 

620 database connection is read-write or the table is a temporary table. 

621 

622 Parameters 

623 ---------- 

624 table : `sqlalchemy.schema.Table` 

625 SQLAlchemy table object to check. 

626 msg : `str`, optional 

627 If provided, raise `ReadOnlyDatabaseError` instead of returning 

628 `False`, with this message. 

629 """ 

630 if not self.isTableWriteable(table): 

631 raise ReadOnlyDatabaseError(msg) 

632 

633 @contextmanager 

634 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]: 

635 """Return a context manager in which the database's static DDL schema 

636 can be declared. 

637 

638 Parameters 

639 ---------- 

640 create : `bool` 

641 If `True`, attempt to create all tables at the end of the context. 

642 If `False`, they will be assumed to already exist. 

643 

644 Returns 

645 ------- 

646 schema : `StaticTablesContext` 

647 A helper object that is used to add new tables. 

648 

649 Raises 

650 ------ 

651 ReadOnlyDatabaseError 

652 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

653 and one or more declared tables do not already exist. 

654 

655 Examples 

656 -------- 

657 Given a `Database` instance ``db``:: 

658 

659 with db.declareStaticTables(create=True) as schema: 

660 schema.addTable("table1", TableSpec(...)) 

661 schema.addTable("table2", TableSpec(...)) 

662 

663 Notes 

664 ----- 

665 A database's static DDL schema must be declared before any dynamic 

666 tables are managed via calls to `ensureTableExists` or 

667 `getExistingTable`. The order in which static schema tables are added 

668 inside the context block is unimportant; they will automatically be 

669 sorted and added in an order consistent with their foreign key 

670 relationships. 

671 """ 

672 if create and not self.isWriteable(): 

673 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

674 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

675 try: 

676 with self._session() as connection: 

677 context = StaticTablesContext(self, connection) 

678 if create and context._tableNames: 

679 # Looks like database is already initalized, to avoid 

680 # danger of modifying/destroying valid schema we refuse to 

681 # do anything in this case 

682 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.") 

683 yield context 

684 for table, foreignKey in context._foreignKeys: 

685 table.append_constraint(foreignKey) 

686 if create: 

687 if self.namespace is not None: 

688 if self.namespace not in context._inspector.get_schema_names(): 

689 with self.transaction(): 

690 connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

691 # In our tables we have columns that make use of sqlalchemy 

692 # Sequence objects. There is currently a bug in sqlalchemy 

693 # that causes a deprecation warning to be thrown on a 

694 # property of the Sequence object when the repr for the 

695 # sequence is created. Here a filter is used to catch these 

696 # deprecation warnings when tables are created. 

697 with warnings.catch_warnings(): 

698 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

699 self._metadata.create_all(self._engine) 

700 # call all initializer methods sequentially 

701 for init in context._initializers: 

702 init(self) 

703 except BaseException: 

704 self._metadata = None 

705 raise 

706 

707 @abstractmethod 

708 def isWriteable(self) -> bool: 

709 """Return `True` if this database can be modified by this client.""" 

710 raise NotImplementedError() 

711 

712 @abstractmethod 

713 def __str__(self) -> str: 

714 """Return a human-readable identifier for this `Database`, including 

715 any namespace or schema that identifies its names within a `Registry`. 

716 """ 

717 raise NotImplementedError() 

718 

719 @property 

720 def dialect(self) -> sqlalchemy.engine.Dialect: 

721 """The SQLAlchemy dialect for this database engine 

722 (`sqlalchemy.engine.Dialect`). 

723 """ 

724 return self._engine.dialect 

725 

726 def shrinkDatabaseEntityName(self, original: str) -> str: 

727 """Return a version of the given name that fits within this database 

728 engine's length limits for table, constraint, indexes, and sequence 

729 names. 

730 

731 Implementations should not assume that simple truncation is safe, 

732 because multiple long names often begin with the same prefix. 

733 

734 The default implementation simply returns the given name. 

735 

736 Parameters 

737 ---------- 

738 original : `str` 

739 The original name. 

740 

741 Returns 

742 ------- 

743 shrunk : `str` 

744 The new, possibly shortened name. 

745 """ 

746 return original 

747 

748 def expandDatabaseEntityName(self, shrunk: str) -> str: 

749 """Retrieve the original name for a database entity that was too long 

750 to fit within the database engine's limits. 

751 

752 Parameters 

753 ---------- 

754 original : `str` 

755 The original name. 

756 

757 Returns 

758 ------- 

759 shrunk : `str` 

760 The new, possibly shortened name. 

761 """ 

762 return shrunk 

763 

764 def _mangleTableName(self, name: str) -> str: 

765 """Map a logical, user-visible table name to the true table name used 

766 in the database. 

767 

768 The default implementation returns the given name unchanged. 

769 

770 Parameters 

771 ---------- 

772 name : `str` 

773 Input table name. Should not include a namespace (i.e. schema) 

774 prefix. 

775 

776 Returns 

777 ------- 

778 mangled : `str` 

779 Mangled version of the table name (still with no namespace prefix). 

780 

781 Notes 

782 ----- 

783 Reimplementations of this method must be idempotent - mangling an 

784 already-mangled name must have no effect. 

785 """ 

786 return name 

787 

788 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]: 

789 """Create constraints based on this spec. 

790 

791 Parameters 

792 ---------- 

793 table : `str` 

794 Name of the table this column is being added to. 

795 spec : `FieldSpec` 

796 Specification for the field to be added. 

797 

798 Returns 

799 ------- 

800 constraint : `list` of `sqlalchemy.CheckConstraint` 

801 Constraint added for this column. 

802 """ 

803 # By default we return no additional constraints 

804 return [] 

805 

806 def _convertFieldSpec( 

807 self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, **kwargs: Any 

808 ) -> sqlalchemy.schema.Column: 

809 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

810 

811 Parameters 

812 ---------- 

813 table : `str` 

814 Name of the table this column is being added to. 

815 spec : `FieldSpec` 

816 Specification for the field to be added. 

817 metadata : `sqlalchemy.MetaData` 

818 SQLAlchemy representation of the DDL schema this field's table is 

819 being added to. 

820 **kwargs 

821 Additional keyword arguments to forward to the 

822 `sqlalchemy.schema.Column` constructor. This is provided to make 

823 it easier for derived classes to delegate to ``super()`` while 

824 making only minor changes. 

825 

826 Returns 

827 ------- 

828 column : `sqlalchemy.schema.Column` 

829 SQLAlchemy representation of the field. 

830 """ 

831 args = [spec.name, spec.getSizedColumnType()] 

832 if spec.autoincrement: 

833 # Generate a sequence to use for auto incrementing for databases 

834 # that do not support it natively. This will be ignored by 

835 # sqlalchemy for databases that do support it. 

836 args.append( 

837 sqlalchemy.Sequence( 

838 self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), metadata=metadata 

839 ) 

840 ) 

841 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

842 return sqlalchemy.schema.Column( 

843 *args, 

844 nullable=spec.nullable, 

845 primary_key=spec.primaryKey, 

846 comment=spec.doc, 

847 server_default=spec.default, 

848 **kwargs, 

849 ) 

850 

851 def _convertForeignKeySpec( 

852 self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, **kwargs: Any 

853 ) -> sqlalchemy.schema.ForeignKeyConstraint: 

854 """Convert a `ForeignKeySpec` to a 

855 `sqlalchemy.schema.ForeignKeyConstraint`. 

856 

857 Parameters 

858 ---------- 

859 table : `str` 

860 Name of the table this foreign key is being added to. 

861 spec : `ForeignKeySpec` 

862 Specification for the foreign key to be added. 

863 metadata : `sqlalchemy.MetaData` 

864 SQLAlchemy representation of the DDL schema this constraint is 

865 being added to. 

866 **kwargs 

867 Additional keyword arguments to forward to the 

868 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

869 provided to make it easier for derived classes to delegate to 

870 ``super()`` while making only minor changes. 

871 

872 Returns 

873 ------- 

874 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

875 SQLAlchemy representation of the constraint. 

876 """ 

877 name = self.shrinkDatabaseEntityName( 

878 "_".join( 

879 ["fkey", table, self._mangleTableName(spec.table)] + list(spec.target) + list(spec.source) 

880 ) 

881 ) 

882 return sqlalchemy.schema.ForeignKeyConstraint( 

883 spec.source, 

884 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

885 name=name, 

886 ondelete=spec.onDelete, 

887 ) 

888 

889 def _convertExclusionConstraintSpec( 

890 self, 

891 table: str, 

892 spec: Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...], 

893 metadata: sqlalchemy.MetaData, 

894 ) -> sqlalchemy.schema.Constraint: 

895 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy 

896 constraint representation. 

897 

898 Parameters 

899 ---------- 

900 table : `str` 

901 Name of the table this constraint is being added to. 

902 spec : `tuple` [ `str` or `type` ] 

903 A tuple of `str` column names and the `type` object returned by 

904 `getTimespanRepresentation` (which must appear exactly once), 

905 indicating the order of the columns in the index used to back the 

906 constraint. 

907 metadata : `sqlalchemy.MetaData` 

908 SQLAlchemy representation of the DDL schema this constraint is 

909 being added to. 

910 

911 Returns 

912 ------- 

913 constraint : `sqlalchemy.schema.Constraint` 

914 SQLAlchemy representation of the constraint. 

915 

916 Raises 

917 ------ 

918 NotImplementedError 

919 Raised if this database does not support exclusion constraints. 

920 """ 

921 raise NotImplementedError(f"Database {self} does not support exclusion constraints.") 

922 

923 def _convertTableSpec( 

924 self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, **kwargs: Any 

925 ) -> sqlalchemy.schema.Table: 

926 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

927 

928 Parameters 

929 ---------- 

930 spec : `TableSpec` 

931 Specification for the foreign key to be added. 

932 metadata : `sqlalchemy.MetaData` 

933 SQLAlchemy representation of the DDL schema this table is being 

934 added to. 

935 **kwargs 

936 Additional keyword arguments to forward to the 

937 `sqlalchemy.schema.Table` constructor. This is provided to make it 

938 easier for derived classes to delegate to ``super()`` while making 

939 only minor changes. 

940 

941 Returns 

942 ------- 

943 table : `sqlalchemy.schema.Table` 

944 SQLAlchemy representation of the table. 

945 

946 Notes 

947 ----- 

948 This method does not handle ``spec.foreignKeys`` at all, in order to 

949 avoid circular dependencies. These are added by higher-level logic in 

950 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

951 """ 

952 name = self._mangleTableName(name) 

953 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

954 

955 # Add any column constraints 

956 for fieldSpec in spec.fields: 

957 args.extend(self._makeColumnConstraints(name, fieldSpec)) 

958 

959 # Track indexes added for primary key and unique constraints, to make 

960 # sure we don't add duplicate explicit or foreign key indexes for 

961 # those. 

962 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)} 

963 args.extend( 

964 sqlalchemy.schema.UniqueConstraint( 

965 *columns, name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

966 ) 

967 for columns in spec.unique 

968 ) 

969 allIndexes.update(spec.unique) 

970 args.extend( 

971 sqlalchemy.schema.Index( 

972 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

973 *columns, 

974 unique=(columns in spec.unique), 

975 ) 

976 for columns in spec.indexes 

977 if columns not in allIndexes 

978 ) 

979 allIndexes.update(spec.indexes) 

980 args.extend( 

981 sqlalchemy.schema.Index( 

982 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)), 

983 *fk.source, 

984 ) 

985 for fk in spec.foreignKeys 

986 if fk.addIndex and fk.source not in allIndexes 

987 ) 

988 

989 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion) 

990 

991 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

992 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwargs) 

993 

994 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

995 """Ensure that a table with the given name and specification exists, 

996 creating it if necessary. 

997 

998 Parameters 

999 ---------- 

1000 name : `str` 

1001 Name of the table (not including namespace qualifiers). 

1002 spec : `TableSpec` 

1003 Specification for the table. This will be used when creating the 

1004 table, and *may* be used when obtaining an existing table to check 

1005 for consistency, but no such check is guaranteed. 

1006 

1007 Returns 

1008 ------- 

1009 table : `sqlalchemy.schema.Table` 

1010 SQLAlchemy representation of the table. 

1011 

1012 Raises 

1013 ------ 

1014 ReadOnlyDatabaseError 

1015 Raised if `isWriteable` returns `False`, and the table does not 

1016 already exist. 

1017 DatabaseConflictError 

1018 Raised if the table exists but ``spec`` is inconsistent with its 

1019 definition. 

1020 

1021 Notes 

1022 ----- 

1023 This method may not be called within transactions. It may be called on 

1024 read-only databases if and only if the table does in fact already 

1025 exist. 

1026 

1027 Subclasses may override this method, but usually should not need to. 

1028 """ 

1029 # TODO: if _engine is used to make a table then it uses separate 

1030 # connection and should not interfere with current transaction 

1031 assert ( 

1032 self._session_connection is None or not self._session_connection.in_transaction() 

1033 ), "Table creation interrupts transactions." 

1034 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

1035 table = self.getExistingTable(name, spec) 

1036 if table is not None: 

1037 return table 

1038 if not self.isWriteable(): 

1039 raise ReadOnlyDatabaseError( 

1040 f"Table {name} does not exist, and cannot be created " 

1041 f"because database {self} is read-only." 

1042 ) 

1043 table = self._convertTableSpec(name, spec, self._metadata) 

1044 for foreignKeySpec in spec.foreignKeys: 

1045 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

1046 try: 

1047 with self._transaction() as (_, connection): 

1048 table.create(connection) 

1049 except sqlalchemy.exc.DatabaseError: 

1050 # Some other process could have created the table meanwhile, which 

1051 # usually causes OperationalError or ProgrammingError. We cannot 

1052 # use IF NOT EXISTS clause in this case due to PostgreSQL race 

1053 # condition on server side which causes IntegrityError. Instead we 

1054 # catch these exceptions (they all inherit DatabaseError) and 

1055 # re-check whether table is now there. 

1056 table = self.getExistingTable(name, spec) 

1057 if table is None: 

1058 raise 

1059 return table 

1060 

1061 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

1062 """Obtain an existing table with the given name and specification. 

1063 

1064 Parameters 

1065 ---------- 

1066 name : `str` 

1067 Name of the table (not including namespace qualifiers). 

1068 spec : `TableSpec` 

1069 Specification for the table. This will be used when creating the 

1070 SQLAlchemy representation of the table, and it is used to 

1071 check that the actual table in the database is consistent. 

1072 

1073 Returns 

1074 ------- 

1075 table : `sqlalchemy.schema.Table` or `None` 

1076 SQLAlchemy representation of the table, or `None` if it does not 

1077 exist. 

1078 

1079 Raises 

1080 ------ 

1081 DatabaseConflictError 

1082 Raised if the table exists but ``spec`` is inconsistent with its 

1083 definition. 

1084 

1085 Notes 

1086 ----- 

1087 This method can be called within transactions and never modifies the 

1088 database. 

1089 

1090 Subclasses may override this method, but usually should not need to. 

1091 """ 

1092 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

1093 name = self._mangleTableName(name) 

1094 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

1095 if table is not None: 

1096 if spec.fields.names != set(table.columns.keys()): 

1097 raise DatabaseConflictError( 

1098 f"Table '{name}' has already been defined differently; the new " 

1099 f"specification has columns {list(spec.fields.names)}, while " 

1100 f"the previous definition has {list(table.columns.keys())}." 

1101 ) 

1102 else: 

1103 inspector = sqlalchemy.inspect( 

1104 self._engine if self._session_connection is None else self._session_connection 

1105 ) 

1106 if name in inspector.get_table_names(schema=self.namespace): 

1107 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

1108 table = self._convertTableSpec(name, spec, self._metadata) 

1109 for foreignKeySpec in spec.foreignKeys: 

1110 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

1111 return table 

1112 return table 

1113 

1114 def _make_temporary_table( 

1115 self, 

1116 connection: sqlalchemy.engine.Connection, 

1117 spec: ddl.TableSpec, 

1118 name: Optional[str] = None, 

1119 **kwargs: Any, 

1120 ) -> sqlalchemy.schema.Table: 

1121 """Create a temporary table. 

1122 

1123 Parameters 

1124 ---------- 

1125 connection : `sqlalchemy.engine.Connection` 

1126 Connection to use when creating the table. 

1127 spec : `TableSpec` 

1128 Specification for the table. 

1129 name : `str`, optional 

1130 A unique (within this session/connetion) name for the table. 

1131 Subclasses may override to modify the actual name used. If not 

1132 provided, a unique name will be generated. 

1133 **kwargs 

1134 Additional keyword arguments to forward to the 

1135 `sqlalchemy.schema.Table` constructor. This is provided to make it 

1136 easier for derived classes to delegate to ``super()`` while making 

1137 only minor changes. 

1138 

1139 Returns 

1140 ------- 

1141 table : `sqlalchemy.schema.Table` 

1142 SQLAlchemy representation of the table. 

1143 """ 

1144 if name is None: 

1145 name = f"tmp_{uuid.uuid4().hex}" 

1146 metadata = self._metadata 

1147 if metadata is None: 

1148 raise RuntimeError("Cannot create temporary table before static schema is defined.") 

1149 table = self._convertTableSpec( 

1150 name, spec, metadata, prefixes=["TEMPORARY"], schema=sqlalchemy.schema.BLANK_SCHEMA, **kwargs 

1151 ) 

1152 if table.key in self._temp_tables: 

1153 if table.key != name: 

1154 raise ValueError( 

1155 f"A temporary table with name {name} (transformed to {table.key} by " 

1156 f"Database) already exists." 

1157 ) 

1158 for foreignKeySpec in spec.foreignKeys: 

1159 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, metadata)) 

1160 table.create(connection) 

1161 return table 

1162 

1163 @classmethod 

1164 def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]: 

1165 """Return a `type` that encapsulates the way `Timespan` objects are 

1166 stored in this database. 

1167 

1168 `Database` does not automatically use the return type of this method 

1169 anywhere else; calling code is responsible for making sure that DDL 

1170 and queries are consistent with it. 

1171 

1172 Returns 

1173 ------- 

1174 TimespanReprClass : `type` (`TimespanDatabaseRepresention` subclass) 

1175 A type that encapsulates the way `Timespan` objects should be 

1176 stored in this database. 

1177 

1178 Notes 

1179 ----- 

1180 There are two big reasons we've decided to keep timespan-mangling logic 

1181 outside the `Database` implementations, even though the choice of 

1182 representation is ultimately up to a `Database` implementation: 

1183 

1184 - Timespans appear in relatively few tables and queries in our 

1185 typical usage, and the code that operates on them is already aware 

1186 that it is working with timespans. In contrast, a 

1187 timespan-representation-aware implementation of, say, `insert`, 

1188 would need to have extra logic to identify when timespan-mangling 

1189 needed to occur, which would usually be useless overhead. 

1190 

1191 - SQLAlchemy's rich SELECT query expression system has no way to wrap 

1192 multiple columns in a single expression object (the ORM does, but 

1193 we are not using the ORM). So we would have to wrap _much_ more of 

1194 that code in our own interfaces to encapsulate timespan 

1195 representations there. 

1196 """ 

1197 return TimespanDatabaseRepresentation.Compound 

1198 

1199 @classmethod 

1200 def getSpatialRegionRepresentation(cls) -> Type[SpatialRegionDatabaseRepresentation]: 

1201 """Return a `type` that encapsulates the way `lsst.sphgeom.Region` 

1202 objects are stored in this database. 

1203 

1204 `Database` does not automatically use the return type of this method 

1205 anywhere else; calling code is responsible for making sure that DDL 

1206 and queries are consistent with it. 

1207 

1208 Returns 

1209 ------- 

1210 RegionReprClass : `type` (`SpatialRegionDatabaseRepresention` subclass) 

1211 A type that encapsulates the way `lsst.sphgeom.Region` objects 

1212 should be stored in this database. 

1213 

1214 Notes 

1215 ----- 

1216 See `getTimespanRepresentation` for comments on why this method is not 

1217 more tightly integrated with the rest of the `Database` interface. 

1218 """ 

1219 return SpatialRegionDatabaseRepresentation 

1220 

1221 def sync( 

1222 self, 

1223 table: sqlalchemy.schema.Table, 

1224 *, 

1225 keys: Dict[str, Any], 

1226 compared: Optional[Dict[str, Any]] = None, 

1227 extra: Optional[Dict[str, Any]] = None, 

1228 returning: Optional[Sequence[str]] = None, 

1229 update: bool = False, 

1230 ) -> Tuple[Optional[Dict[str, Any]], Union[bool, Dict[str, Any]]]: 

1231 """Insert into a table as necessary to ensure database contains 

1232 values equivalent to the given ones. 

1233 

1234 Parameters 

1235 ---------- 

1236 table : `sqlalchemy.schema.Table` 

1237 Table to be queried and possibly inserted into. 

1238 keys : `dict` 

1239 Column name-value pairs used to search for an existing row; must 

1240 be a combination that can be used to select a single row if one 

1241 exists. If such a row does not exist, these values are used in 

1242 the insert. 

1243 compared : `dict`, optional 

1244 Column name-value pairs that are compared to those in any existing 

1245 row. If such a row does not exist, these rows are used in the 

1246 insert. 

1247 extra : `dict`, optional 

1248 Column name-value pairs that are ignored if a matching row exists, 

1249 but used in an insert if one is necessary. 

1250 returning : `~collections.abc.Sequence` of `str`, optional 

1251 The names of columns whose values should be returned. 

1252 update : `bool`, optional 

1253 If `True` (`False` is default), update the existing row with the 

1254 values in ``compared`` instead of raising `DatabaseConflictError`. 

1255 

1256 Returns 

1257 ------- 

1258 row : `dict`, optional 

1259 The value of the fields indicated by ``returning``, or `None` if 

1260 ``returning`` is `None`. 

1261 inserted_or_updated : `bool` or `dict` 

1262 If `True`, a new row was inserted; if `False`, a matching row 

1263 already existed. If a `dict` (only possible if ``update=True``), 

1264 then an existing row was updated, and the dict maps the names of 

1265 the updated columns to their *old* values (new values can be 

1266 obtained from ``compared``). 

1267 

1268 Raises 

1269 ------ 

1270 DatabaseConflictError 

1271 Raised if the values in ``compared`` do not match the values in the 

1272 database. 

1273 ReadOnlyDatabaseError 

1274 Raised if `isWriteable` returns `False`, and no matching record 

1275 already exists. 

1276 

1277 Notes 

1278 ----- 

1279 May be used inside transaction contexts, so implementations may not 

1280 perform operations that interrupt transactions. 

1281 

1282 It may be called on read-only databases if and only if the matching row 

1283 does in fact already exist. 

1284 """ 

1285 

1286 def check() -> Tuple[int, Optional[Dict[str, Any]], Optional[List]]: 

1287 """Query for a row that matches the ``key`` argument, and compare 

1288 to what was given by the caller. 

1289 

1290 Returns 

1291 ------- 

1292 n : `int` 

1293 Number of matching rows. ``n != 1`` is always an error, but 

1294 it's a different kind of error depending on where `check` is 

1295 being called. 

1296 bad : `dict` or `None` 

1297 The subset of the keys of ``compared`` for which the existing 

1298 values did not match the given one, mapped to the existing 

1299 values in the database. Once again, ``not bad`` is always an 

1300 error, but a different kind on context. `None` if ``n != 1`` 

1301 result : `list` or `None` 

1302 Results in the database that correspond to the columns given 

1303 in ``returning``, or `None` if ``returning is None``. 

1304 """ 

1305 toSelect: Set[str] = set() 

1306 if compared is not None: 

1307 toSelect.update(compared.keys()) 

1308 if returning is not None: 

1309 toSelect.update(returning) 

1310 if not toSelect: 

1311 # Need to select some column, even if we just want to see 

1312 # how many rows we get back. 

1313 toSelect.add(next(iter(keys.keys()))) 

1314 selectSql = ( 

1315 sqlalchemy.sql.select(*[table.columns[k].label(k) for k in toSelect]) 

1316 .select_from(table) 

1317 .where(sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])) 

1318 ) 

1319 with self._transaction() as (_, connection): 

1320 fetched = list(connection.execute(selectSql).mappings()) 

1321 if len(fetched) != 1: 

1322 return len(fetched), None, None 

1323 existing = fetched[0] 

1324 if compared is not None: 

1325 

1326 def safeNotEqual(a: Any, b: Any) -> bool: 

1327 if isinstance(a, astropy.time.Time): 

1328 return not time_utils.TimeConverter().times_equal(a, b) 

1329 return a != b 

1330 

1331 inconsistencies = { 

1332 k: existing[k] for k, v in compared.items() if safeNotEqual(existing[k], v) 

1333 } 

1334 else: 

1335 inconsistencies = {} 

1336 if returning is not None: 

1337 toReturn: Optional[list] = [existing[k] for k in returning] 

1338 else: 

1339 toReturn = None 

1340 return 1, inconsistencies, toReturn 

1341 

1342 def format_bad(inconsistencies: Dict[str, Any]) -> str: 

1343 """Format the 'bad' dictionary of existing values returned by 

1344 ``check`` into a string suitable for an error message. 

1345 """ 

1346 assert compared is not None, "Should not be able to get inconsistencies without comparing." 

1347 return ", ".join(f"{k}: {v!r} != {compared[k]!r}" for k, v in inconsistencies.items()) 

1348 

1349 if self.isTableWriteable(table): 

1350 # Try an insert first, but allow it to fail (in only specific 

1351 # ways). 

1352 row = keys.copy() 

1353 if compared is not None: 

1354 row.update(compared) 

1355 if extra is not None: 

1356 row.update(extra) 

1357 with self.transaction(): 

1358 inserted = bool(self.ensure(table, row)) 

1359 inserted_or_updated: Union[bool, Dict[str, Any]] 

1360 # Need to perform check() for this branch inside the 

1361 # transaction, so we roll back an insert that didn't do 

1362 # what we expected. That limits the extent to which we 

1363 # can reduce duplication between this block and the other 

1364 # ones that perform similar logic. 

1365 n, bad, result = check() 

1366 if n < 1: 

1367 raise ConflictingDefinitionError( 

1368 f"Attempted to ensure {row} exists by inserting it with ON CONFLICT IGNORE, " 

1369 f"but a post-insert query on {keys} returned no results. " 

1370 f"Insert was {'' if inserted else 'not '}reported as successful. " 

1371 "This can occur if the insert violated a database constraint other than the " 

1372 "unique constraint or primary key used to identify the row in this call." 

1373 ) 

1374 elif n > 1: 

1375 raise RuntimeError( 

1376 f"Keys passed to sync {keys.keys()} do not comprise a " 

1377 f"unique constraint for table {table.name}." 

1378 ) 

1379 elif bad: 

1380 assert ( 

1381 compared is not None 

1382 ), "Should not be able to get inconsistencies without comparing." 

1383 if inserted: 

1384 raise RuntimeError( 

1385 f"Conflict ({bad}) in sync after successful insert; this is " 

1386 "possible if the same table is being updated by a concurrent " 

1387 "process that isn't using sync, but it may also be a bug in " 

1388 "daf_butler." 

1389 ) 

1390 elif update: 

1391 with self._transaction() as (_, connection): 

1392 connection.execute( 

1393 table.update() 

1394 .where(sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()])) 

1395 .values(**{k: compared[k] for k in bad.keys()}) 

1396 ) 

1397 inserted_or_updated = bad 

1398 else: 

1399 raise DatabaseConflictError( 

1400 f"Conflict in sync for table {table.name} on column(s) {format_bad(bad)}." 

1401 ) 

1402 else: 

1403 inserted_or_updated = inserted 

1404 else: 

1405 # Database is not writeable; just see if the row exists. 

1406 n, bad, result = check() 

1407 if n < 1: 

1408 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

1409 elif n > 1: 

1410 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

1411 elif bad: 

1412 if update: 

1413 raise ReadOnlyDatabaseError("sync needs to update, but database is read-only.") 

1414 else: 

1415 raise DatabaseConflictError( 

1416 f"Conflict in sync for table {table.name} on column(s) {format_bad(bad)}." 

1417 ) 

1418 inserted_or_updated = False 

1419 if returning is None: 

1420 return None, inserted_or_updated 

1421 else: 

1422 assert result is not None 

1423 return {k: v for k, v in zip(returning, result)}, inserted_or_updated 

1424 

1425 def insert( 

1426 self, 

1427 table: sqlalchemy.schema.Table, 

1428 *rows: dict, 

1429 returnIds: bool = False, 

1430 select: Optional[sqlalchemy.sql.Select] = None, 

1431 names: Optional[Iterable[str]] = None, 

1432 ) -> Optional[List[int]]: 

1433 """Insert one or more rows into a table, optionally returning 

1434 autoincrement primary key values. 

1435 

1436 Parameters 

1437 ---------- 

1438 table : `sqlalchemy.schema.Table` 

1439 Table rows should be inserted into. 

1440 returnIds: `bool` 

1441 If `True` (`False` is default), return the values of the table's 

1442 autoincrement primary key field (which much exist). 

1443 select : `sqlalchemy.sql.Select`, optional 

1444 A SELECT query expression to insert rows from. Cannot be provided 

1445 with either ``rows`` or ``returnIds=True``. 

1446 names : `Iterable` [ `str` ], optional 

1447 Names of columns in ``table`` to be populated, ordered to match the 

1448 columns returned by ``select``. Ignored if ``select`` is `None`. 

1449 If not provided, the columns returned by ``select`` must be named 

1450 to match the desired columns of ``table``. 

1451 *rows 

1452 Positional arguments are the rows to be inserted, as dictionaries 

1453 mapping column name to value. The keys in all dictionaries must 

1454 be the same. 

1455 

1456 Returns 

1457 ------- 

1458 ids : `None`, or `list` of `int` 

1459 If ``returnIds`` is `True`, a `list` containing the inserted 

1460 values for the table's autoincrement primary key. 

1461 

1462 Raises 

1463 ------ 

1464 ReadOnlyDatabaseError 

1465 Raised if `isWriteable` returns `False` when this method is called. 

1466 

1467 Notes 

1468 ----- 

1469 The default implementation uses bulk insert syntax when ``returnIds`` 

1470 is `False`, and a loop over single-row insert operations when it is 

1471 `True`. 

1472 

1473 Derived classes should reimplement when they can provide a more 

1474 efficient implementation (especially for the latter case). 

1475 

1476 May be used inside transaction contexts, so implementations may not 

1477 perform operations that interrupt transactions. 

1478 """ 

1479 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.") 

1480 if select is not None and (rows or returnIds): 

1481 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.") 

1482 if not rows and select is None: 

1483 if returnIds: 

1484 return [] 

1485 else: 

1486 return None 

1487 with self._transaction() as (_, connection): 

1488 if not returnIds: 

1489 if select is not None: 

1490 if names is None: 

1491 # columns() is deprecated since 1.4, but 

1492 # selected_columns() method did not exist in 1.3. 

1493 if hasattr(select, "selected_columns"): 

1494 names = select.selected_columns.keys() 

1495 else: 

1496 names = select.columns.keys() 

1497 connection.execute(table.insert().from_select(names, select)) 

1498 else: 

1499 connection.execute(table.insert(), rows) 

1500 return None 

1501 else: 

1502 sql = table.insert() 

1503 return [connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

1504 

1505 @abstractmethod 

1506 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None: 

1507 """Insert one or more rows into a table, replacing any existing rows 

1508 for which insertion of a new row would violate the primary key 

1509 constraint. 

1510 

1511 Parameters 

1512 ---------- 

1513 table : `sqlalchemy.schema.Table` 

1514 Table rows should be inserted into. 

1515 *rows 

1516 Positional arguments are the rows to be inserted, as dictionaries 

1517 mapping column name to value. The keys in all dictionaries must 

1518 be the same. 

1519 

1520 Raises 

1521 ------ 

1522 ReadOnlyDatabaseError 

1523 Raised if `isWriteable` returns `False` when this method is called. 

1524 

1525 Notes 

1526 ----- 

1527 May be used inside transaction contexts, so implementations may not 

1528 perform operations that interrupt transactions. 

1529 

1530 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

1531 exception when a constraint other than the primary key would be 

1532 violated. 

1533 

1534 Implementations are not required to support `replace` on tables 

1535 with autoincrement keys. 

1536 """ 

1537 raise NotImplementedError() 

1538 

1539 @abstractmethod 

1540 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict, primary_key_only: bool = False) -> int: 

1541 """Insert one or more rows into a table, skipping any rows for which 

1542 insertion would violate a unique constraint. 

1543 

1544 Parameters 

1545 ---------- 

1546 table : `sqlalchemy.schema.Table` 

1547 Table rows should be inserted into. 

1548 *rows 

1549 Positional arguments are the rows to be inserted, as dictionaries 

1550 mapping column name to value. The keys in all dictionaries must 

1551 be the same. 

1552 primary_key_only : `bool`, optional 

1553 If `True` (`False` is default), only skip rows that violate the 

1554 primary key constraint, and raise an exception (and rollback 

1555 transactions) for other constraint violations. 

1556 

1557 Returns 

1558 ------- 

1559 count : `int` 

1560 The number of rows actually inserted. 

1561 

1562 Raises 

1563 ------ 

1564 ReadOnlyDatabaseError 

1565 Raised if `isWriteable` returns `False` when this method is called. 

1566 This is raised even if the operation would do nothing even on a 

1567 writeable database. 

1568 

1569 Notes 

1570 ----- 

1571 May be used inside transaction contexts, so implementations may not 

1572 perform operations that interrupt transactions. 

1573 

1574 Implementations are not required to support `ensure` on tables 

1575 with autoincrement keys. 

1576 """ 

1577 raise NotImplementedError() 

1578 

1579 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

1580 """Delete one or more rows from a table. 

1581 

1582 Parameters 

1583 ---------- 

1584 table : `sqlalchemy.schema.Table` 

1585 Table that rows should be deleted from. 

1586 columns: `~collections.abc.Iterable` of `str` 

1587 The names of columns that will be used to constrain the rows to 

1588 be deleted; these will be combined via ``AND`` to form the 

1589 ``WHERE`` clause of the delete query. 

1590 *rows 

1591 Positional arguments are the keys of rows to be deleted, as 

1592 dictionaries mapping column name to value. The keys in all 

1593 dictionaries must be exactly the names in ``columns``. 

1594 

1595 Returns 

1596 ------- 

1597 count : `int` 

1598 Number of rows deleted. 

1599 

1600 Raises 

1601 ------ 

1602 ReadOnlyDatabaseError 

1603 Raised if `isWriteable` returns `False` when this method is called. 

1604 

1605 Notes 

1606 ----- 

1607 May be used inside transaction contexts, so implementations may not 

1608 perform operations that interrupt transactions. 

1609 

1610 The default implementation should be sufficient for most derived 

1611 classes. 

1612 """ 

1613 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.") 

1614 if columns and not rows: 

1615 # If there are no columns, this operation is supposed to delete 

1616 # everything (so we proceed as usual). But if there are columns, 

1617 # but no rows, it was a constrained bulk operation where the 

1618 # constraint is that no rows match, and we should short-circuit 

1619 # while reporting that no rows were affected. 

1620 return 0 

1621 sql = table.delete() 

1622 columns = list(columns) # Force iterators to list 

1623 

1624 # More efficient to use IN operator if there is only one 

1625 # variable changing across all rows. 

1626 content: Dict[str, Set] = defaultdict(set) 

1627 if len(columns) == 1: 

1628 # Nothing to calculate since we can always use IN 

1629 column = columns[0] 

1630 changing_columns = [column] 

1631 content[column] = set(row[column] for row in rows) 

1632 else: 

1633 for row in rows: 

1634 for k, v in row.items(): 

1635 content[k].add(v) 

1636 changing_columns = [col for col, values in content.items() if len(values) > 1] 

1637 

1638 if len(changing_columns) != 1: 

1639 # More than one column changes each time so do explicit bind 

1640 # parameters and have each row processed separately. 

1641 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

1642 if whereTerms: 

1643 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

1644 with self._transaction() as (_, connection): 

1645 return connection.execute(sql, rows).rowcount 

1646 else: 

1647 # One of the columns has changing values but any others are 

1648 # fixed. In this case we can use an IN operator and be more 

1649 # efficient. 

1650 name = changing_columns.pop() 

1651 

1652 # Simple where clause for the unchanging columns 

1653 clauses = [] 

1654 for k, v in content.items(): 

1655 if k == name: 

1656 continue 

1657 column = table.columns[k] 

1658 # The set only has one element 

1659 clauses.append(column == v.pop()) 

1660 

1661 # The IN operator will not work for "infinite" numbers of 

1662 # rows so must batch it up into distinct calls. 

1663 in_content = list(content[name]) 

1664 n_elements = len(in_content) 

1665 

1666 rowcount = 0 

1667 iposn = 0 

1668 n_per_loop = 1_000 # Controls how many items to put in IN clause 

1669 with self._transaction() as (_, connection): 

1670 for iposn in range(0, n_elements, n_per_loop): 

1671 endpos = iposn + n_per_loop 

1672 in_clause = table.columns[name].in_(in_content[iposn:endpos]) 

1673 

1674 newsql = sql.where(sqlalchemy.sql.and_(*clauses, in_clause)) 

1675 rowcount += connection.execute(newsql).rowcount 

1676 return rowcount 

1677 

1678 def deleteWhere(self, table: sqlalchemy.schema.Table, where: sqlalchemy.sql.ClauseElement) -> int: 

1679 """Delete rows from a table with pre-constructed WHERE clause. 

1680 

1681 Parameters 

1682 ---------- 

1683 table : `sqlalchemy.schema.Table` 

1684 Table that rows should be deleted from. 

1685 where: `sqlalchemy.sql.ClauseElement` 

1686 The names of columns that will be used to constrain the rows to 

1687 be deleted; these will be combined via ``AND`` to form the 

1688 ``WHERE`` clause of the delete query. 

1689 

1690 Returns 

1691 ------- 

1692 count : `int` 

1693 Number of rows deleted. 

1694 

1695 Raises 

1696 ------ 

1697 ReadOnlyDatabaseError 

1698 Raised if `isWriteable` returns `False` when this method is called. 

1699 

1700 Notes 

1701 ----- 

1702 May be used inside transaction contexts, so implementations may not 

1703 perform operations that interrupt transactions. 

1704 

1705 The default implementation should be sufficient for most derived 

1706 classes. 

1707 """ 

1708 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.") 

1709 

1710 sql = table.delete().where(where) 

1711 with self._transaction() as (_, connection): 

1712 return connection.execute(sql).rowcount 

1713 

1714 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

1715 """Update one or more rows in a table. 

1716 

1717 Parameters 

1718 ---------- 

1719 table : `sqlalchemy.schema.Table` 

1720 Table containing the rows to be updated. 

1721 where : `dict` [`str`, `str`] 

1722 A mapping from the names of columns that will be used to search for 

1723 existing rows to the keys that will hold these values in the 

1724 ``rows`` dictionaries. Note that these may not be the same due to 

1725 SQLAlchemy limitations. 

1726 *rows 

1727 Positional arguments are the rows to be updated. The keys in all 

1728 dictionaries must be the same, and may correspond to either a 

1729 value in the ``where`` dictionary or the name of a column to be 

1730 updated. 

1731 

1732 Returns 

1733 ------- 

1734 count : `int` 

1735 Number of rows matched (regardless of whether the update actually 

1736 modified them). 

1737 

1738 Raises 

1739 ------ 

1740 ReadOnlyDatabaseError 

1741 Raised if `isWriteable` returns `False` when this method is called. 

1742 

1743 Notes 

1744 ----- 

1745 May be used inside transaction contexts, so implementations may not 

1746 perform operations that interrupt transactions. 

1747 

1748 The default implementation should be sufficient for most derived 

1749 classes. 

1750 """ 

1751 self.assertTableWriteable(table, f"Cannot update read-only table {table}.") 

1752 if not rows: 

1753 return 0 

1754 sql = table.update().where( 

1755 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1756 ) 

1757 with self._transaction() as (_, connection): 

1758 return connection.execute(sql, rows).rowcount 

1759 

1760 @contextmanager 

1761 def query( 

1762 self, sql: sqlalchemy.sql.expression.SelectBase, *args: Any, **kwargs: Any 

1763 ) -> Iterator[sqlalchemy.engine.CursorResult]: 

1764 """Run a SELECT query against the database. 

1765 

1766 Parameters 

1767 ---------- 

1768 sql : `sqlalchemy.sql.expression.SelectBase` 

1769 A SQLAlchemy representation of a ``SELECT`` query. 

1770 *args 

1771 Additional positional arguments are forwarded to 

1772 `sqlalchemy.engine.Connection.execute`. 

1773 **kwargs 

1774 Additional keyword arguments are forwarded to 

1775 `sqlalchemy.engine.Connection.execute`. 

1776 

1777 Returns 

1778 ------- 

1779 result_context : `sqlalchemy.engine.CursorResults` 

1780 Context manager that returns the query result object when entered. 

1781 These results are invalidated when the context is exited. 

1782 """ 

1783 if self._session_connection is None: 

1784 connection = self._engine.connect() 

1785 else: 

1786 connection = self._session_connection 

1787 result = connection.execute(sql, *args, **kwargs) 

1788 try: 

1789 yield result 

1790 finally: 

1791 if connection is not self._session_connection: 

1792 connection.close() 

1793 

1794 origin: int 

1795 """An integer ID that should be used as the default for any datasets, 

1796 quanta, or other entities that use a (autoincrement, origin) compound 

1797 primary key (`int`). 

1798 """ 

1799 

1800 namespace: Optional[str] 

1801 """The schema or namespace this database instance is associated with 

1802 (`str` or `None`). 

1803 """