Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "SchemaAlreadyDefinedError", 

28 "StaticTablesContext", 

29] 

30 

31from abc import ABC, abstractmethod 

32from contextlib import contextmanager 

33from typing import ( 

34 Any, 

35 Callable, 

36 Dict, 

37 Iterable, 

38 Iterator, 

39 List, 

40 Optional, 

41 Sequence, 

42 Set, 

43 Tuple, 

44 Type, 

45 Union, 

46) 

47import uuid 

48import warnings 

49 

50import astropy.time 

51import sqlalchemy 

52 

53from ...core import SpatialRegionDatabaseRepresentation, TimespanDatabaseRepresentation, ddl, time_utils 

54from .._exceptions import ConflictingDefinitionError 

55 

56_IN_SAVEPOINT_TRANSACTION = "IN_SAVEPOINT_TRANSACTION" 

57 

58 

59def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None: 

60 """Test that the definition of a table in a `ddl.TableSpec` and from 

61 database introspection are consistent. 

62 

63 Parameters 

64 ---------- 

65 name : `str` 

66 Name of the table (only used in error messages). 

67 spec : `ddl.TableSpec` 

68 Specification of the table. 

69 inspection : `dict` 

70 Dictionary returned by 

71 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

72 

73 Raises 

74 ------ 

75 DatabaseConflictError 

76 Raised if the definitions are inconsistent. 

77 """ 

78 columnNames = [c["name"] for c in inspection] 

79 if spec.fields.names != set(columnNames): 

80 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

81 f"specification has columns {list(spec.fields.names)}, while the " 

82 f"table in the database has {columnNames}.") 

83 

84 

85class ReadOnlyDatabaseError(RuntimeError): 

86 """Exception raised when a write operation is called on a read-only 

87 `Database`. 

88 """ 

89 

90 

91class DatabaseConflictError(ConflictingDefinitionError): 

92 """Exception raised when database content (row values or schema entities) 

93 are inconsistent with what this client expects. 

94 """ 

95 

96 

97class SchemaAlreadyDefinedError(RuntimeError): 

98 """Exception raised when trying to initialize database schema when some 

99 tables already exist. 

100 """ 

101 

102 

103class StaticTablesContext: 

104 """Helper class used to declare the static schema for a registry layer 

105 in a database. 

106 

107 An instance of this class is returned by `Database.declareStaticTables`, 

108 which should be the only way it should be constructed. 

109 """ 

110 

111 def __init__(self, db: Database): 

112 self._db = db 

113 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = [] 

114 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

115 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

116 self._initializers: List[Callable[[Database], None]] = [] 

117 

118 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

119 """Add a new table to the schema, returning its sqlalchemy 

120 representation. 

121 

122 The new table may not actually be created until the end of the 

123 context created by `Database.declareStaticTables`, allowing tables 

124 to be declared in any order even in the presence of foreign key 

125 relationships. 

126 """ 

127 name = self._db._mangleTableName(name) 

128 if name in self._tableNames: 

129 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

130 schema=self._db.namespace)) 

131 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

132 for foreignKeySpec in spec.foreignKeys: 

133 self._foreignKeys.append( 

134 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

135 ) 

136 return table 

137 

138 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

139 """Add a named tuple of tables to the schema, returning their 

140 SQLAlchemy representations in a named tuple of the same type. 

141 

142 The new tables may not actually be created until the end of the 

143 context created by `Database.declareStaticTables`, allowing tables 

144 to be declared in any order even in the presence of foreign key 

145 relationships. 

146 

147 Notes 

148 ----- 

149 ``specs`` *must* be an instance of a type created by 

150 `collections.namedtuple`, not just regular tuple, and the returned 

151 object is guaranteed to be the same. Because `~collections.namedtuple` 

152 is just a factory for `type` objects, not an actual type itself, 

153 we cannot represent this with type annotations. 

154 """ 

155 return specs._make(self.addTable(name, spec) # type: ignore 

156 for name, spec in zip(specs._fields, specs)) # type: ignore 

157 

158 def addInitializer(self, initializer: Callable[[Database], None]) -> None: 

159 """Add a method that does one-time initialization of a database. 

160 

161 Initialization can mean anything that changes state of a database 

162 and needs to be done exactly once after database schema was created. 

163 An example for that could be population of schema attributes. 

164 

165 Parameters 

166 ---------- 

167 initializer : callable 

168 Method of a single argument which is a `Database` instance. 

169 """ 

170 self._initializers.append(initializer) 

171 

172 

173class Database(ABC): 

174 """An abstract interface that represents a particular database engine's 

175 representation of a single schema/namespace/database. 

176 

177 Parameters 

178 ---------- 

179 origin : `int` 

180 An integer ID that should be used as the default for any datasets, 

181 quanta, or other entities that use a (autoincrement, origin) compound 

182 primary key. 

183 connection : `sqlalchemy.engine.Connection` 

184 The SQLAlchemy connection this `Database` wraps. 

185 namespace : `str`, optional 

186 Name of the schema or namespace this instance is associated with. 

187 This is passed as the ``schema`` argument when constructing a 

188 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

189 avoid confusion between "schema means namespace" and "schema means 

190 table definitions". 

191 

192 Notes 

193 ----- 

194 `Database` requires all write operations to go through its special named 

195 methods. Our write patterns are sufficiently simple that we don't really 

196 need the full flexibility of SQL insert/update/delete syntax, and we need 

197 non-standard (but common) functionality in these operations sufficiently 

198 often that it seems worthwhile to provide our own generic API. 

199 

200 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

201 their SQLAlchemy representation) to be run, as we expect these to require 

202 significantly more sophistication while still being limited to standard 

203 SQL. 

204 

205 `Database` itself has several underscore-prefixed attributes: 

206 

207 - ``_connection``: SQLAlchemy object representing the connection. 

208 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

209 the tables and other schema entities. 

210 

211 These are considered protected (derived classes may access them, but other 

212 code should not), and read-only, aside from executing SQL via 

213 ``_connection``. 

214 """ 

215 

216 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection, 

217 namespace: Optional[str] = None): 

218 self.origin = origin 

219 self.namespace = namespace 

220 self._connection = connection 

221 self._metadata: Optional[sqlalchemy.schema.MetaData] = None 

222 self._tempTables: Set[str] = set() 

223 

224 def __repr__(self) -> str: 

225 # Rather than try to reproduce all the parameters used to create 

226 # the object, instead report the more useful information of the 

227 # connection URL. 

228 uri = str(self._connection.engine.url) 

229 if self.namespace: 

230 uri += f"#{self.namespace}" 

231 return f'{type(self).__name__}("{uri}")' 

232 

233 @classmethod 

234 def makeDefaultUri(cls, root: str) -> Optional[str]: 

235 """Create a default connection URI appropriate for the given root 

236 directory, or `None` if there can be no such default. 

237 """ 

238 return None 

239 

240 @classmethod 

241 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

242 writeable: bool = True) -> Database: 

243 """Construct a database from a SQLAlchemy URI. 

244 

245 Parameters 

246 ---------- 

247 uri : `str` 

248 A SQLAlchemy URI connection string. 

249 origin : `int` 

250 An integer ID that should be used as the default for any datasets, 

251 quanta, or other entities that use a (autoincrement, origin) 

252 compound primary key. 

253 namespace : `str`, optional 

254 A database namespace (i.e. schema) the new instance should be 

255 associated with. If `None` (default), the namespace (if any) is 

256 inferred from the URI. 

257 writeable : `bool`, optional 

258 If `True`, allow write operations on the database, including 

259 ``CREATE TABLE``. 

260 

261 Returns 

262 ------- 

263 db : `Database` 

264 A new `Database` instance. 

265 """ 

266 return cls.fromConnection(cls.connect(uri, writeable=writeable), 

267 origin=origin, 

268 namespace=namespace, 

269 writeable=writeable) 

270 

271 @classmethod 

272 @abstractmethod 

273 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection: 

274 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI. 

275 

276 Parameters 

277 ---------- 

278 uri : `str` 

279 A SQLAlchemy URI connection string. 

280 origin : `int` 

281 An integer ID that should be used as the default for any datasets, 

282 quanta, or other entities that use a (autoincrement, origin) 

283 compound primary key. 

284 writeable : `bool`, optional 

285 If `True`, allow write operations on the database, including 

286 ``CREATE TABLE``. 

287 

288 Returns 

289 ------- 

290 connection : `sqlalchemy.engine.Connection` 

291 A database connection. 

292 

293 Notes 

294 ----- 

295 Subclasses that support other ways to connect to a database are 

296 encouraged to add optional arguments to their implementation of this 

297 method, as long as they maintain compatibility with the base class 

298 call signature. 

299 """ 

300 raise NotImplementedError() 

301 

302 @classmethod 

303 @abstractmethod 

304 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int, 

305 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

306 """Create a new `Database` from an existing 

307 `sqlalchemy.engine.Connection`. 

308 

309 Parameters 

310 ---------- 

311 connection : `sqllachemy.engine.Connection` 

312 The connection for the the database. May be shared between 

313 `Database` instances. 

314 origin : `int` 

315 An integer ID that should be used as the default for any datasets, 

316 quanta, or other entities that use a (autoincrement, origin) 

317 compound primary key. 

318 namespace : `str`, optional 

319 A different database namespace (i.e. schema) the new instance 

320 should be associated with. If `None` (default), the namespace 

321 (if any) is inferred from the connection. 

322 writeable : `bool`, optional 

323 If `True`, allow write operations on the database, including 

324 ``CREATE TABLE``. 

325 

326 Returns 

327 ------- 

328 db : `Database` 

329 A new `Database` instance. 

330 

331 Notes 

332 ----- 

333 This method allows different `Database` instances to share the same 

334 connection, which is desirable when they represent different namespaces 

335 can be queried together. This also ties their transaction state, 

336 however; starting a transaction in any database automatically starts 

337 on in all other databases. 

338 """ 

339 raise NotImplementedError() 

340 

341 @contextmanager 

342 def transaction(self, *, interrupting: bool = False, savepoint: bool = False, 

343 lock: Iterable[sqlalchemy.schema.Table] = ()) -> Iterator: 

344 """Return a context manager that represents a transaction. 

345 

346 Parameters 

347 ---------- 

348 interrupting : `bool`, optional 

349 If `True` (`False` is default), this transaction block may not be 

350 nested without an outer one, and attempting to do so is a logic 

351 (i.e. assertion) error. 

352 savepoint : `bool`, optional 

353 If `True` (`False` is default), create a `SAVEPOINT`, allowing 

354 exceptions raised by the database (e.g. due to constraint 

355 violations) during this transaction's context to be caught outside 

356 it without also rolling back all operations in an outer transaction 

357 block. If `False`, transactions may still be nested, but a 

358 rollback may be generated at any level and affects all levels, and 

359 commits are deferred until the outermost block completes. If any 

360 outer transaction block was created with ``savepoint=True``, all 

361 inner blocks will be as well (regardless of the actual value 

362 passed). This has no effect if this is the outermost transaction. 

363 lock : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

364 A list of tables to lock for the duration of this transaction. 

365 These locks are guaranteed to prevent concurrent writes and allow 

366 this transaction (only) to acquire the same locks (others should 

367 block), but only prevent concurrent reads if the database engine 

368 requires that in order to block concurrent writes. 

369 

370 Notes 

371 ----- 

372 All transactions on a connection managed by one or more `Database` 

373 instances _must_ go through this method, or transaction state will not 

374 be correctly managed. 

375 """ 

376 assert not (interrupting and self._connection.in_transaction()), ( 

377 "Logic error in transaction nesting: an operation that would " 

378 "interrupt the active transaction context has been requested." 

379 ) 

380 # We remember whether we are already in a SAVEPOINT transaction via the 

381 # connection object's 'info' dict, which is explicitly for user 

382 # information like this. This is safer than a regular `Database` 

383 # instance attribute, because it guards against multiple `Database` 

384 # instances sharing the same connection. The need to use our own flag 

385 # here to track whether we're in a nested transaction should go away in 

386 # SQLAlchemy 1.4, which seems to have a 

387 # `Connection.in_nested_transaction()` method. 

388 savepoint = savepoint or self._connection.info.get(_IN_SAVEPOINT_TRANSACTION, False) 

389 self._connection.info[_IN_SAVEPOINT_TRANSACTION] = savepoint 

390 if self._connection.in_transaction() and savepoint: 

391 trans = self._connection.begin_nested() 

392 else: 

393 # Use a regular (non-savepoint) transaction always for the 

394 # outermost context, as well as when a savepoint was not requested. 

395 trans = self._connection.begin() 

396 self._lockTables(lock) 

397 try: 

398 yield 

399 trans.commit() 

400 except BaseException: 

401 trans.rollback() 

402 raise 

403 finally: 

404 if not self._connection.in_transaction(): 

405 self._connection.info.pop(_IN_SAVEPOINT_TRANSACTION, None) 

406 

407 @abstractmethod 

408 def _lockTables(self, tables: Iterable[sqlalchemy.schema.Table] = ()) -> None: 

409 """Acquire locks on the given tables. 

410 

411 This is an implementation hook for subclasses, called by `transaction`. 

412 It should not be called directly by other code. 

413 

414 Parameters 

415 ---------- 

416 tables : `Iterable` [ `sqlalchemy.schema.Table` ], optional 

417 A list of tables to lock for the duration of this transaction. 

418 These locks are guaranteed to prevent concurrent writes and allow 

419 this transaction (only) to acquire the same locks (others should 

420 block), but only prevent concurrent reads if the database engine 

421 requires that in order to block concurrent writes. 

422 """ 

423 raise NotImplementedError() 

424 

425 def isTableWriteable(self, table: sqlalchemy.schema.Table) -> bool: 

426 """Check whether a table is writeable, either because the database 

427 connection is read-write or the table is a temporary table. 

428 

429 Parameters 

430 ---------- 

431 table : `sqlalchemy.schema.Table` 

432 SQLAlchemy table object to check. 

433 

434 Returns 

435 ------- 

436 writeable : `bool` 

437 Whether this table is writeable. 

438 """ 

439 return self.isWriteable() or table.key in self._tempTables 

440 

441 def assertTableWriteable(self, table: sqlalchemy.schema.Table, msg: str) -> None: 

442 """Raise if the given table is not writeable, either because the 

443 database connection is read-write or the table is a temporary table. 

444 

445 Parameters 

446 ---------- 

447 table : `sqlalchemy.schema.Table` 

448 SQLAlchemy table object to check. 

449 msg : `str`, optional 

450 If provided, raise `ReadOnlyDatabaseError` instead of returning 

451 `False`, with this message. 

452 """ 

453 if not self.isTableWriteable(table): 

454 raise ReadOnlyDatabaseError(msg) 

455 

456 @contextmanager 

457 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]: 

458 """Return a context manager in which the database's static DDL schema 

459 can be declared. 

460 

461 Parameters 

462 ---------- 

463 create : `bool` 

464 If `True`, attempt to create all tables at the end of the context. 

465 If `False`, they will be assumed to already exist. 

466 

467 Returns 

468 ------- 

469 schema : `StaticTablesContext` 

470 A helper object that is used to add new tables. 

471 

472 Raises 

473 ------ 

474 ReadOnlyDatabaseError 

475 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

476 and one or more declared tables do not already exist. 

477 

478 Examples 

479 -------- 

480 Given a `Database` instance ``db``:: 

481 

482 with db.declareStaticTables(create=True) as schema: 

483 schema.addTable("table1", TableSpec(...)) 

484 schema.addTable("table2", TableSpec(...)) 

485 

486 Notes 

487 ----- 

488 A database's static DDL schema must be declared before any dynamic 

489 tables are managed via calls to `ensureTableExists` or 

490 `getExistingTable`. The order in which static schema tables are added 

491 inside the context block is unimportant; they will automatically be 

492 sorted and added in an order consistent with their foreign key 

493 relationships. 

494 """ 

495 if create and not self.isWriteable(): 

496 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

497 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

498 try: 

499 context = StaticTablesContext(self) 

500 if create and context._tableNames: 

501 # Looks like database is already initalized, to avoid danger 

502 # of modifying/destroying valid schema we refuse to do 

503 # anything in this case 

504 raise SchemaAlreadyDefinedError(f"Cannot create tables in non-empty database {self}.") 

505 yield context 

506 for table, foreignKey in context._foreignKeys: 

507 table.append_constraint(foreignKey) 

508 if create: 

509 if self.namespace is not None: 

510 if self.namespace not in context._inspector.get_schema_names(): 

511 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

512 # In our tables we have columns that make use of sqlalchemy 

513 # Sequence objects. There is currently a bug in sqlalchemy that 

514 # causes a deprecation warning to be thrown on a property of 

515 # the Sequence object when the repr for the sequence is 

516 # created. Here a filter is used to catch these deprecation 

517 # warnings when tables are created. 

518 with warnings.catch_warnings(): 

519 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

520 self._metadata.create_all(self._connection) 

521 # call all initializer methods sequentially 

522 for init in context._initializers: 

523 init(self) 

524 except BaseException: 

525 self._metadata = None 

526 raise 

527 

528 @abstractmethod 

529 def isWriteable(self) -> bool: 

530 """Return `True` if this database can be modified by this client. 

531 """ 

532 raise NotImplementedError() 

533 

534 @abstractmethod 

535 def __str__(self) -> str: 

536 """Return a human-readable identifier for this `Database`, including 

537 any namespace or schema that identifies its names within a `Registry`. 

538 """ 

539 raise NotImplementedError() 

540 

541 @property 

542 def dialect(self) -> sqlalchemy.engine.Dialect: 

543 """The SQLAlchemy dialect for this database engine 

544 (`sqlalchemy.engine.Dialect`). 

545 """ 

546 return self._connection.dialect 

547 

548 def shrinkDatabaseEntityName(self, original: str) -> str: 

549 """Return a version of the given name that fits within this database 

550 engine's length limits for table, constraint, indexes, and sequence 

551 names. 

552 

553 Implementations should not assume that simple truncation is safe, 

554 because multiple long names often begin with the same prefix. 

555 

556 The default implementation simply returns the given name. 

557 

558 Parameters 

559 ---------- 

560 original : `str` 

561 The original name. 

562 

563 Returns 

564 ------- 

565 shrunk : `str` 

566 The new, possibly shortened name. 

567 """ 

568 return original 

569 

570 def expandDatabaseEntityName(self, shrunk: str) -> str: 

571 """Retrieve the original name for a database entity that was too long 

572 to fit within the database engine's limits. 

573 

574 Parameters 

575 ---------- 

576 original : `str` 

577 The original name. 

578 

579 Returns 

580 ------- 

581 shrunk : `str` 

582 The new, possibly shortened name. 

583 """ 

584 return shrunk 

585 

586 def _mangleTableName(self, name: str) -> str: 

587 """Map a logical, user-visible table name to the true table name used 

588 in the database. 

589 

590 The default implementation returns the given name unchanged. 

591 

592 Parameters 

593 ---------- 

594 name : `str` 

595 Input table name. Should not include a namespace (i.e. schema) 

596 prefix. 

597 

598 Returns 

599 ------- 

600 mangled : `str` 

601 Mangled version of the table name (still with no namespace prefix). 

602 

603 Notes 

604 ----- 

605 Reimplementations of this method must be idempotent - mangling an 

606 already-mangled name must have no effect. 

607 """ 

608 return name 

609 

610 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]: 

611 """Create constraints based on this spec. 

612 

613 Parameters 

614 ---------- 

615 table : `str` 

616 Name of the table this column is being added to. 

617 spec : `FieldSpec` 

618 Specification for the field to be added. 

619 

620 Returns 

621 ------- 

622 constraint : `list` of `sqlalchemy.CheckConstraint` 

623 Constraint added for this column. 

624 """ 

625 # By default we return no additional constraints 

626 return [] 

627 

628 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

629 **kwds: Any) -> sqlalchemy.schema.Column: 

630 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

631 

632 Parameters 

633 ---------- 

634 table : `str` 

635 Name of the table this column is being added to. 

636 spec : `FieldSpec` 

637 Specification for the field to be added. 

638 metadata : `sqlalchemy.MetaData` 

639 SQLAlchemy representation of the DDL schema this field's table is 

640 being added to. 

641 **kwds 

642 Additional keyword arguments to forward to the 

643 `sqlalchemy.schema.Column` constructor. This is provided to make 

644 it easier for derived classes to delegate to ``super()`` while 

645 making only minor changes. 

646 

647 Returns 

648 ------- 

649 column : `sqlalchemy.schema.Column` 

650 SQLAlchemy representation of the field. 

651 """ 

652 args = [spec.name, spec.getSizedColumnType()] 

653 if spec.autoincrement: 

654 # Generate a sequence to use for auto incrementing for databases 

655 # that do not support it natively. This will be ignored by 

656 # sqlalchemy for databases that do support it. 

657 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

658 metadata=metadata)) 

659 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

660 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

661 comment=spec.doc, server_default=spec.default, **kwds) 

662 

663 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

664 **kwds: Any) -> sqlalchemy.schema.ForeignKeyConstraint: 

665 """Convert a `ForeignKeySpec` to a 

666 `sqlalchemy.schema.ForeignKeyConstraint`. 

667 

668 Parameters 

669 ---------- 

670 table : `str` 

671 Name of the table this foreign key is being added to. 

672 spec : `ForeignKeySpec` 

673 Specification for the foreign key to be added. 

674 metadata : `sqlalchemy.MetaData` 

675 SQLAlchemy representation of the DDL schema this constraint is 

676 being added to. 

677 **kwds 

678 Additional keyword arguments to forward to the 

679 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

680 provided to make it easier for derived classes to delegate to 

681 ``super()`` while making only minor changes. 

682 

683 Returns 

684 ------- 

685 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

686 SQLAlchemy representation of the constraint. 

687 """ 

688 name = self.shrinkDatabaseEntityName( 

689 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

690 + list(spec.target) + list(spec.source)) 

691 ) 

692 return sqlalchemy.schema.ForeignKeyConstraint( 

693 spec.source, 

694 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

695 name=name, 

696 ondelete=spec.onDelete 

697 ) 

698 

699 def _convertExclusionConstraintSpec(self, table: str, 

700 spec: Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...], 

701 metadata: sqlalchemy.MetaData) -> sqlalchemy.schema.Constraint: 

702 """Convert a `tuple` from `ddl.TableSpec.exclusion` into a SQLAlchemy 

703 constraint representation. 

704 

705 Parameters 

706 ---------- 

707 table : `str` 

708 Name of the table this constraint is being added to. 

709 spec : `tuple` [ `str` or `type` ] 

710 A tuple of `str` column names and the `type` object returned by 

711 `getTimespanRepresentation` (which must appear exactly once), 

712 indicating the order of the columns in the index used to back the 

713 constraint. 

714 metadata : `sqlalchemy.MetaData` 

715 SQLAlchemy representation of the DDL schema this constraint is 

716 being added to. 

717 

718 Returns 

719 ------- 

720 constraint : `sqlalchemy.schema.Constraint` 

721 SQLAlchemy representation of the constraint. 

722 

723 Raises 

724 ------ 

725 NotImplementedError 

726 Raised if this database does not support exclusion constraints. 

727 """ 

728 raise NotImplementedError(f"Database {self} does not support exclusion constraints.") 

729 

730 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

731 **kwds: Any) -> sqlalchemy.schema.Table: 

732 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

733 

734 Parameters 

735 ---------- 

736 spec : `TableSpec` 

737 Specification for the foreign key to be added. 

738 metadata : `sqlalchemy.MetaData` 

739 SQLAlchemy representation of the DDL schema this table is being 

740 added to. 

741 **kwds 

742 Additional keyword arguments to forward to the 

743 `sqlalchemy.schema.Table` constructor. This is provided to make it 

744 easier for derived classes to delegate to ``super()`` while making 

745 only minor changes. 

746 

747 Returns 

748 ------- 

749 table : `sqlalchemy.schema.Table` 

750 SQLAlchemy representation of the table. 

751 

752 Notes 

753 ----- 

754 This method does not handle ``spec.foreignKeys`` at all, in order to 

755 avoid circular dependencies. These are added by higher-level logic in 

756 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

757 """ 

758 name = self._mangleTableName(name) 

759 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

760 

761 # Add any column constraints 

762 for fieldSpec in spec.fields: 

763 args.extend(self._makeColumnConstraints(name, fieldSpec)) 

764 

765 # Track indexes added for primary key and unique constraints, to make 

766 # sure we don't add duplicate explicit or foreign key indexes for 

767 # those. 

768 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)} 

769 args.extend( 

770 sqlalchemy.schema.UniqueConstraint( 

771 *columns, 

772 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

773 ) 

774 for columns in spec.unique 

775 ) 

776 allIndexes.update(spec.unique) 

777 args.extend( 

778 sqlalchemy.schema.Index( 

779 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

780 *columns, 

781 unique=(columns in spec.unique) 

782 ) 

783 for columns in spec.indexes if columns not in allIndexes 

784 ) 

785 allIndexes.update(spec.indexes) 

786 args.extend( 

787 sqlalchemy.schema.Index( 

788 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)), 

789 *fk.source, 

790 ) 

791 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes 

792 ) 

793 

794 args.extend(self._convertExclusionConstraintSpec(name, excl, metadata) for excl in spec.exclusion) 

795 

796 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

797 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

798 

799 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

800 """Ensure that a table with the given name and specification exists, 

801 creating it if necessary. 

802 

803 Parameters 

804 ---------- 

805 name : `str` 

806 Name of the table (not including namespace qualifiers). 

807 spec : `TableSpec` 

808 Specification for the table. This will be used when creating the 

809 table, and *may* be used when obtaining an existing table to check 

810 for consistency, but no such check is guaranteed. 

811 

812 Returns 

813 ------- 

814 table : `sqlalchemy.schema.Table` 

815 SQLAlchemy representation of the table. 

816 

817 Raises 

818 ------ 

819 ReadOnlyDatabaseError 

820 Raised if `isWriteable` returns `False`, and the table does not 

821 already exist. 

822 DatabaseConflictError 

823 Raised if the table exists but ``spec`` is inconsistent with its 

824 definition. 

825 

826 Notes 

827 ----- 

828 This method may not be called within transactions. It may be called on 

829 read-only databases if and only if the table does in fact already 

830 exist. 

831 

832 Subclasses may override this method, but usually should not need to. 

833 """ 

834 assert not self._connection.in_transaction(), "Table creation interrupts transactions." 

835 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

836 table = self.getExistingTable(name, spec) 

837 if table is not None: 

838 return table 

839 if not self.isWriteable(): 

840 raise ReadOnlyDatabaseError( 

841 f"Table {name} does not exist, and cannot be created " 

842 f"because database {self} is read-only." 

843 ) 

844 table = self._convertTableSpec(name, spec, self._metadata) 

845 for foreignKeySpec in spec.foreignKeys: 

846 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

847 table.create(self._connection) 

848 return table 

849 

850 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

851 """Obtain an existing table with the given name and specification. 

852 

853 Parameters 

854 ---------- 

855 name : `str` 

856 Name of the table (not including namespace qualifiers). 

857 spec : `TableSpec` 

858 Specification for the table. This will be used when creating the 

859 SQLAlchemy representation of the table, and it is used to 

860 check that the actual table in the database is consistent. 

861 

862 Returns 

863 ------- 

864 table : `sqlalchemy.schema.Table` or `None` 

865 SQLAlchemy representation of the table, or `None` if it does not 

866 exist. 

867 

868 Raises 

869 ------ 

870 DatabaseConflictError 

871 Raised if the table exists but ``spec`` is inconsistent with its 

872 definition. 

873 

874 Notes 

875 ----- 

876 This method can be called within transactions and never modifies the 

877 database. 

878 

879 Subclasses may override this method, but usually should not need to. 

880 """ 

881 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

882 name = self._mangleTableName(name) 

883 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

884 if table is not None: 

885 if spec.fields.names != set(table.columns.keys()): 

886 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

887 f"specification has columns {list(spec.fields.names)}, while " 

888 f"the previous definition has {list(table.columns.keys())}.") 

889 else: 

890 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

891 if name in inspector.get_table_names(schema=self.namespace): 

892 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

893 table = self._convertTableSpec(name, spec, self._metadata) 

894 for foreignKeySpec in spec.foreignKeys: 

895 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

896 return table 

897 return table 

898 

899 def makeTemporaryTable(self, spec: ddl.TableSpec, name: Optional[str] = None) -> sqlalchemy.schema.Table: 

900 """Create a temporary table. 

901 

902 Parameters 

903 ---------- 

904 spec : `TableSpec` 

905 Specification for the table. 

906 name : `str`, optional 

907 A unique (within this session/connetion) name for the table. 

908 Subclasses may override to modify the actual name used. If not 

909 provided, a unique name will be generated. 

910 

911 Returns 

912 ------- 

913 table : `sqlalchemy.schema.Table` 

914 SQLAlchemy representation of the table. 

915 

916 Notes 

917 ----- 

918 Temporary tables may be created, dropped, and written to even in 

919 read-only databases - at least according to the Python-level 

920 protections in the `Database` classes. Server permissions may say 

921 otherwise, but in that case they probably need to be modified to 

922 support the full range of expected read-only butler behavior. 

923 

924 Temporary table rows are guaranteed to be dropped when a connection is 

925 closed. `Database` implementations are permitted to allow the table to 

926 remain as long as this is transparent to the user (i.e. "creating" the 

927 temporary table in a new session should not be an error, even if it 

928 does nothing). 

929 

930 It may not be possible to use temporary tables within transactions with 

931 some database engines (or configurations thereof). 

932 """ 

933 if name is None: 

934 name = f"tmp_{uuid.uuid4().hex}" 

935 table = self._convertTableSpec(name, spec, self._metadata, prefixes=['TEMPORARY'], 

936 schema=sqlalchemy.schema.BLANK_SCHEMA) 

937 if table.key in self._tempTables: 

938 if table.key != name: 

939 raise ValueError(f"A temporary table with name {name} (transformed to {table.key} by " 

940 f"Database) already exists.") 

941 for foreignKeySpec in spec.foreignKeys: 

942 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

943 table.create(self._connection) 

944 self._tempTables.add(table.key) 

945 return table 

946 

947 def dropTemporaryTable(self, table: sqlalchemy.schema.Table) -> None: 

948 """Drop a temporary table. 

949 

950 Parameters 

951 ---------- 

952 table : `sqlalchemy.schema.Table` 

953 A SQLAlchemy object returned by a previous call to 

954 `makeTemporaryTable`. 

955 """ 

956 if table.key in self._tempTables: 

957 table.drop(self._connection) 

958 self._tempTables.remove(table.key) 

959 else: 

960 raise TypeError(f"Table {table.key} was not created by makeTemporaryTable.") 

961 

962 @classmethod 

963 def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]: 

964 """Return a `type` that encapsulates the way `Timespan` objects are 

965 stored in this database. 

966 

967 `Database` does not automatically use the return type of this method 

968 anywhere else; calling code is responsible for making sure that DDL 

969 and queries are consistent with it. 

970 

971 Returns 

972 ------- 

973 TimespanReprClass : `type` (`TimespanDatabaseRepresention` subclass) 

974 A type that encapsulates the way `Timespan` objects should be 

975 stored in this database. 

976 

977 Notes 

978 ----- 

979 There are two big reasons we've decided to keep timespan-mangling logic 

980 outside the `Database` implementations, even though the choice of 

981 representation is ultimately up to a `Database` implementation: 

982 

983 - Timespans appear in relatively few tables and queries in our 

984 typical usage, and the code that operates on them is already aware 

985 that it is working with timespans. In contrast, a 

986 timespan-representation-aware implementation of, say, `insert`, 

987 would need to have extra logic to identify when timespan-mangling 

988 needed to occur, which would usually be useless overhead. 

989 

990 - SQLAlchemy's rich SELECT query expression system has no way to wrap 

991 multiple columns in a single expression object (the ORM does, but 

992 we are not using the ORM). So we would have to wrap _much_ more of 

993 that code in our own interfaces to encapsulate timespan 

994 representations there. 

995 """ 

996 return TimespanDatabaseRepresentation.Compound 

997 

998 @classmethod 

999 def getSpatialRegionRepresentation(cls) -> Type[SpatialRegionDatabaseRepresentation]: 

1000 """Return a `type` that encapsulates the way `lsst.sphgeom.Region` 

1001 objects are stored in this database. 

1002 

1003 `Database` does not automatically use the return type of this method 

1004 anywhere else; calling code is responsible for making sure that DDL 

1005 and queries are consistent with it. 

1006 

1007 Returns 

1008 ------- 

1009 RegionReprClass : `type` (`SpatialRegionDatabaseRepresention` subclass) 

1010 A type that encapsulates the way `lsst.sphgeom.Region` objects 

1011 should be stored in this database. 

1012 

1013 Notes 

1014 ----- 

1015 See `getTimespanRepresentation` for comments on why this method is not 

1016 more tightly integrated with the rest of the `Database` interface. 

1017 """ 

1018 return SpatialRegionDatabaseRepresentation 

1019 

1020 def sync(self, table: sqlalchemy.schema.Table, *, 

1021 keys: Dict[str, Any], 

1022 compared: Optional[Dict[str, Any]] = None, 

1023 extra: Optional[Dict[str, Any]] = None, 

1024 returning: Optional[Sequence[str]] = None, 

1025 ) -> Tuple[Optional[Dict[str, Any]], bool]: 

1026 """Insert into a table as necessary to ensure database contains 

1027 values equivalent to the given ones. 

1028 

1029 Parameters 

1030 ---------- 

1031 table : `sqlalchemy.schema.Table` 

1032 Table to be queried and possibly inserted into. 

1033 keys : `dict` 

1034 Column name-value pairs used to search for an existing row; must 

1035 be a combination that can be used to select a single row if one 

1036 exists. If such a row does not exist, these values are used in 

1037 the insert. 

1038 compared : `dict`, optional 

1039 Column name-value pairs that are compared to those in any existing 

1040 row. If such a row does not exist, these rows are used in the 

1041 insert. 

1042 extra : `dict`, optional 

1043 Column name-value pairs that are ignored if a matching row exists, 

1044 but used in an insert if one is necessary. 

1045 returning : `~collections.abc.Sequence` of `str`, optional 

1046 The names of columns whose values should be returned. 

1047 

1048 Returns 

1049 ------- 

1050 row : `dict`, optional 

1051 The value of the fields indicated by ``returning``, or `None` if 

1052 ``returning`` is `None`. 

1053 inserted : `bool` 

1054 If `True`, a new row was inserted. 

1055 

1056 Raises 

1057 ------ 

1058 DatabaseConflictError 

1059 Raised if the values in ``compared`` do not match the values in the 

1060 database. 

1061 ReadOnlyDatabaseError 

1062 Raised if `isWriteable` returns `False`, and no matching record 

1063 already exists. 

1064 

1065 Notes 

1066 ----- 

1067 May be used inside transaction contexts, so implementations may not 

1068 perform operations that interrupt transactions. 

1069 

1070 It may be called on read-only databases if and only if the matching row 

1071 does in fact already exist. 

1072 """ 

1073 

1074 def check() -> Tuple[int, Optional[List[str]], Optional[List]]: 

1075 """Query for a row that matches the ``key`` argument, and compare 

1076 to what was given by the caller. 

1077 

1078 Returns 

1079 ------- 

1080 n : `int` 

1081 Number of matching rows. ``n != 1`` is always an error, but 

1082 it's a different kind of error depending on where `check` is 

1083 being called. 

1084 bad : `list` of `str`, or `None` 

1085 The subset of the keys of ``compared`` for which the existing 

1086 values did not match the given one. Once again, ``not bad`` 

1087 is always an error, but a different kind on context. `None` 

1088 if ``n != 1`` 

1089 result : `list` or `None` 

1090 Results in the database that correspond to the columns given 

1091 in ``returning``, or `None` if ``returning is None``. 

1092 """ 

1093 toSelect: Set[str] = set() 

1094 if compared is not None: 

1095 toSelect.update(compared.keys()) 

1096 if returning is not None: 

1097 toSelect.update(returning) 

1098 if not toSelect: 

1099 # Need to select some column, even if we just want to see 

1100 # how many rows we get back. 

1101 toSelect.add(next(iter(keys.keys()))) 

1102 selectSql = sqlalchemy.sql.select( 

1103 [table.columns[k].label(k) for k in toSelect] 

1104 ).select_from(table).where( 

1105 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

1106 ) 

1107 fetched = list(self._connection.execute(selectSql).fetchall()) 

1108 if len(fetched) != 1: 

1109 return len(fetched), None, None 

1110 existing = fetched[0] 

1111 if compared is not None: 

1112 

1113 def safeNotEqual(a: Any, b: Any) -> bool: 

1114 if isinstance(a, astropy.time.Time): 

1115 return not time_utils.TimeConverter().times_equal(a, b) 

1116 return a != b 

1117 

1118 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}" 

1119 for k, v in compared.items() 

1120 if safeNotEqual(existing[k], v)] 

1121 else: 

1122 inconsistencies = [] 

1123 if returning is not None: 

1124 toReturn: Optional[list] = [existing[k] for k in returning] 

1125 else: 

1126 toReturn = None 

1127 return 1, inconsistencies, toReturn 

1128 

1129 if self.isTableWriteable(table): 

1130 # Try an insert first, but allow it to fail (in only specific 

1131 # ways). 

1132 row = keys.copy() 

1133 if compared is not None: 

1134 row.update(compared) 

1135 if extra is not None: 

1136 row.update(extra) 

1137 with self.transaction(lock=[table]): 

1138 inserted = bool(self.ensure(table, row)) 

1139 # Need to perform check() for this branch inside the 

1140 # transaction, so we roll back an insert that didn't do 

1141 # what we expected. That limits the extent to which we 

1142 # can reduce duplication between this block and the other 

1143 # ones that perform similar logic. 

1144 n, bad, result = check() 

1145 if n < 1: 

1146 raise RuntimeError( 

1147 "Necessary insertion in sync did not seem to affect table. This is a bug." 

1148 ) 

1149 elif n > 1: 

1150 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

1151 f"unique constraint for table {table.name}.") 

1152 elif bad: 

1153 if inserted: 

1154 raise RuntimeError( 

1155 f"Conflict ({bad}) in sync after successful insert; this is " 

1156 "possible if the same table is being updated by a concurrent " 

1157 "process that isn't using sync, but it may also be a bug in " 

1158 "daf_butler." 

1159 ) 

1160 else: 

1161 raise DatabaseConflictError( 

1162 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1163 ) 

1164 else: 

1165 # Database is not writeable; just see if the row exists. 

1166 n, bad, result = check() 

1167 if n < 1: 

1168 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

1169 elif n > 1: 

1170 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

1171 elif bad: 

1172 raise DatabaseConflictError( 

1173 f"Conflict in sync for table {table.name} on column(s) {bad}." 

1174 ) 

1175 inserted = False 

1176 if returning is None: 

1177 return None, inserted 

1178 else: 

1179 assert result is not None 

1180 return {k: v for k, v in zip(returning, result)}, inserted 

1181 

1182 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

1183 select: Optional[sqlalchemy.sql.Select] = None, 

1184 names: Optional[Iterable[str]] = None, 

1185 ) -> Optional[List[int]]: 

1186 """Insert one or more rows into a table, optionally returning 

1187 autoincrement primary key values. 

1188 

1189 Parameters 

1190 ---------- 

1191 table : `sqlalchemy.schema.Table` 

1192 Table rows should be inserted into. 

1193 returnIds: `bool` 

1194 If `True` (`False` is default), return the values of the table's 

1195 autoincrement primary key field (which much exist). 

1196 select : `sqlalchemy.sql.Select`, optional 

1197 A SELECT query expression to insert rows from. Cannot be provided 

1198 with either ``rows`` or ``returnIds=True``. 

1199 names : `Iterable` [ `str` ], optional 

1200 Names of columns in ``table`` to be populated, ordered to match the 

1201 columns returned by ``select``. Ignored if ``select`` is `None`. 

1202 If not provided, the columns returned by ``select`` must be named 

1203 to match the desired columns of ``table``. 

1204 *rows 

1205 Positional arguments are the rows to be inserted, as dictionaries 

1206 mapping column name to value. The keys in all dictionaries must 

1207 be the same. 

1208 

1209 Returns 

1210 ------- 

1211 ids : `None`, or `list` of `int` 

1212 If ``returnIds`` is `True`, a `list` containing the inserted 

1213 values for the table's autoincrement primary key. 

1214 

1215 Raises 

1216 ------ 

1217 ReadOnlyDatabaseError 

1218 Raised if `isWriteable` returns `False` when this method is called. 

1219 

1220 Notes 

1221 ----- 

1222 The default implementation uses bulk insert syntax when ``returnIds`` 

1223 is `False`, and a loop over single-row insert operations when it is 

1224 `True`. 

1225 

1226 Derived classes should reimplement when they can provide a more 

1227 efficient implementation (especially for the latter case). 

1228 

1229 May be used inside transaction contexts, so implementations may not 

1230 perform operations that interrupt transactions. 

1231 """ 

1232 self.assertTableWriteable(table, f"Cannot insert into read-only table {table}.") 

1233 if select is not None and (rows or returnIds): 

1234 raise TypeError("'select' is incompatible with passing value rows or returnIds=True.") 

1235 if not rows and select is None: 

1236 if returnIds: 

1237 return [] 

1238 else: 

1239 return None 

1240 if not returnIds: 

1241 if select is not None: 

1242 if names is None: 

1243 names = select.columns.keys() 

1244 self._connection.execute(table.insert().from_select(names, select)) 

1245 else: 

1246 self._connection.execute(table.insert(), *rows) 

1247 return None 

1248 else: 

1249 sql = table.insert() 

1250 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

1251 

1252 @abstractmethod 

1253 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None: 

1254 """Insert one or more rows into a table, replacing any existing rows 

1255 for which insertion of a new row would violate the primary key 

1256 constraint. 

1257 

1258 Parameters 

1259 ---------- 

1260 table : `sqlalchemy.schema.Table` 

1261 Table rows should be inserted into. 

1262 *rows 

1263 Positional arguments are the rows to be inserted, as dictionaries 

1264 mapping column name to value. The keys in all dictionaries must 

1265 be the same. 

1266 

1267 Raises 

1268 ------ 

1269 ReadOnlyDatabaseError 

1270 Raised if `isWriteable` returns `False` when this method is called. 

1271 

1272 Notes 

1273 ----- 

1274 May be used inside transaction contexts, so implementations may not 

1275 perform operations that interrupt transactions. 

1276 

1277 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

1278 exception when a constraint other than the primary key would be 

1279 violated. 

1280 

1281 Implementations are not required to support `replace` on tables 

1282 with autoincrement keys. 

1283 """ 

1284 raise NotImplementedError() 

1285 

1286 @abstractmethod 

1287 def ensure(self, table: sqlalchemy.schema.Table, *rows: dict) -> int: 

1288 """Insert one or more rows into a table, skipping any rows for which 

1289 insertion would violate any constraint. 

1290 

1291 Parameters 

1292 ---------- 

1293 table : `sqlalchemy.schema.Table` 

1294 Table rows should be inserted into. 

1295 *rows 

1296 Positional arguments are the rows to be inserted, as dictionaries 

1297 mapping column name to value. The keys in all dictionaries must 

1298 be the same. 

1299 

1300 Returns 

1301 ------- 

1302 count : `int` 

1303 The number of rows actually inserted. 

1304 

1305 Raises 

1306 ------ 

1307 ReadOnlyDatabaseError 

1308 Raised if `isWriteable` returns `False` when this method is called. 

1309 This is raised even if the operation would do nothing even on a 

1310 writeable database. 

1311 

1312 Notes 

1313 ----- 

1314 May be used inside transaction contexts, so implementations may not 

1315 perform operations that interrupt transactions. 

1316 

1317 Implementations are not required to support `ensure` on tables 

1318 with autoincrement keys. 

1319 """ 

1320 raise NotImplementedError() 

1321 

1322 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

1323 """Delete one or more rows from a table. 

1324 

1325 Parameters 

1326 ---------- 

1327 table : `sqlalchemy.schema.Table` 

1328 Table that rows should be deleted from. 

1329 columns: `~collections.abc.Iterable` of `str` 

1330 The names of columns that will be used to constrain the rows to 

1331 be deleted; these will be combined via ``AND`` to form the 

1332 ``WHERE`` clause of the delete query. 

1333 *rows 

1334 Positional arguments are the keys of rows to be deleted, as 

1335 dictionaries mapping column name to value. The keys in all 

1336 dictionaries must exactly the names in ``columns``. 

1337 

1338 Returns 

1339 ------- 

1340 count : `int` 

1341 Number of rows deleted. 

1342 

1343 Raises 

1344 ------ 

1345 ReadOnlyDatabaseError 

1346 Raised if `isWriteable` returns `False` when this method is called. 

1347 

1348 Notes 

1349 ----- 

1350 May be used inside transaction contexts, so implementations may not 

1351 perform operations that interrupt transactions. 

1352 

1353 The default implementation should be sufficient for most derived 

1354 classes. 

1355 """ 

1356 self.assertTableWriteable(table, f"Cannot delete from read-only table {table}.") 

1357 if columns and not rows: 

1358 # If there are no columns, this operation is supposed to delete 

1359 # everything (so we proceed as usual). But if there are columns, 

1360 # but no rows, it was a constrained bulk operation where the 

1361 # constraint is that no rows match, and we should short-circuit 

1362 # while reporting that no rows were affected. 

1363 return 0 

1364 sql = table.delete() 

1365 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

1366 if whereTerms: 

1367 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

1368 return self._connection.execute(sql, *rows).rowcount 

1369 

1370 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

1371 """Update one or more rows in a table. 

1372 

1373 Parameters 

1374 ---------- 

1375 table : `sqlalchemy.schema.Table` 

1376 Table containing the rows to be updated. 

1377 where : `dict` [`str`, `str`] 

1378 A mapping from the names of columns that will be used to search for 

1379 existing rows to the keys that will hold these values in the 

1380 ``rows`` dictionaries. Note that these may not be the same due to 

1381 SQLAlchemy limitations. 

1382 *rows 

1383 Positional arguments are the rows to be updated. The keys in all 

1384 dictionaries must be the same, and may correspond to either a 

1385 value in the ``where`` dictionary or the name of a column to be 

1386 updated. 

1387 

1388 Returns 

1389 ------- 

1390 count : `int` 

1391 Number of rows matched (regardless of whether the update actually 

1392 modified them). 

1393 

1394 Raises 

1395 ------ 

1396 ReadOnlyDatabaseError 

1397 Raised if `isWriteable` returns `False` when this method is called. 

1398 

1399 Notes 

1400 ----- 

1401 May be used inside transaction contexts, so implementations may not 

1402 perform operations that interrupt transactions. 

1403 

1404 The default implementation should be sufficient for most derived 

1405 classes. 

1406 """ 

1407 self.assertTableWriteable(table, f"Cannot update read-only table {table}.") 

1408 if not rows: 

1409 return 0 

1410 sql = table.update().where( 

1411 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1412 ) 

1413 return self._connection.execute(sql, *rows).rowcount 

1414 

1415 def query(self, sql: sqlalchemy.sql.FromClause, 

1416 *args: Any, **kwds: Any) -> sqlalchemy.engine.ResultProxy: 

1417 """Run a SELECT query against the database. 

1418 

1419 Parameters 

1420 ---------- 

1421 sql : `sqlalchemy.sql.FromClause` 

1422 A SQLAlchemy representation of a ``SELECT`` query. 

1423 *args 

1424 Additional positional arguments are forwarded to 

1425 `sqlalchemy.engine.Connection.execute`. 

1426 **kwds 

1427 Additional keyword arguments are forwarded to 

1428 `sqlalchemy.engine.Connection.execute`. 

1429 

1430 Returns 

1431 ------- 

1432 result : `sqlalchemy.engine.ResultProxy` 

1433 Query results. 

1434 

1435 Notes 

1436 ----- 

1437 The default implementation should be sufficient for most derived 

1438 classes. 

1439 """ 

1440 # TODO: should we guard against non-SELECT queries here? 

1441 return self._connection.execute(sql, *args, **kwds) 

1442 

1443 origin: int 

1444 """An integer ID that should be used as the default for any datasets, 

1445 quanta, or other entities that use a (autoincrement, origin) compound 

1446 primary key (`int`). 

1447 """ 

1448 

1449 namespace: Optional[str] 

1450 """The schema or namespace this database instance is associated with 

1451 (`str` or `None`). 

1452 """