Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "StaticTablesContext", 

28] 

29 

30from abc import ABC, abstractmethod 

31from contextlib import contextmanager 

32from typing import ( 

33 Any, 

34 Dict, 

35 Iterable, 

36 List, 

37 Optional, 

38 Sequence, 

39 Tuple, 

40) 

41import warnings 

42 

43import astropy.time 

44import sqlalchemy 

45 

46from ...core import ddl, time_utils 

47 

48 

49def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: Dict[str, Any]): 

50 """Test that the definition of a table in a `ddl.TableSpec` and from 

51 database introspection are consistent. 

52 

53 Parameters 

54 ---------- 

55 name : `str` 

56 Name of the table (only used in error messages). 

57 spec : `ddl.TableSpec` 

58 Specification of the table. 

59 inspection : `dict` 

60 Dictionary returned by 

61 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

62 

63 Raises 

64 ------ 

65 DatabaseConflictError 

66 Raised if the definitions are inconsistent. 

67 """ 

68 columnNames = [c["name"] for c in inspection] 

69 if spec.fields.names != set(columnNames): 

70 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

71 f"specification has columns {list(spec.fields.names)}, while the " 

72 f"table in the database has {columnNames}.") 

73 

74 

75class ReadOnlyDatabaseError(RuntimeError): 

76 """Exception raised when a write operation is called on a read-only 

77 `Database`. 

78 """ 

79 

80 

81class DatabaseConflictError(RuntimeError): 

82 """Exception raised when database content (row values or schema entities) 

83 are inconsistent with what this client expects. 

84 """ 

85 

86 

87class StaticTablesContext: 

88 """Helper class used to declare the static schema for a registry layer 

89 in a database. 

90 

91 An instance of this class is returned by `Database.declareStaticTables`, 

92 which should be the only way it should be constructed. 

93 """ 

94 

95 def __init__(self, db: Database): 

96 self._db = db 

97 self._foreignKeys = [] 

98 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

99 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

100 

101 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

102 """Add a new table to the schema, returning its sqlalchemy 

103 representation. 

104 

105 The new table may not actually be created until the end of the 

106 context created by `Database.declareStaticTables`, allowing tables 

107 to be declared in any order even in the presence of foreign key 

108 relationships. 

109 """ 

110 name = self._db._mangleTableName(name) 

111 if name in self._tableNames: 

112 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

113 schema=self._db.namespace)) 

114 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

115 for foreignKeySpec in spec.foreignKeys: 

116 self._foreignKeys.append( 

117 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

118 ) 

119 return table 

120 

121 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

122 """Add a named tuple of tables to the schema, returning their 

123 SQLAlchemy representations in a named tuple of the same type. 

124 

125 The new tables may not actually be created until the end of the 

126 context created by `Database.declareStaticTables`, allowing tables 

127 to be declared in any order even in the presence of foreign key 

128 relationships. 

129 

130 Notes 

131 ----- 

132 ``specs`` *must* be an instance of a type created by 

133 `collections.namedtuple`, not just regular tuple, and the returned 

134 object is guaranteed to be the same. Because `~collections.namedtuple` 

135 is just a factory for `type` objects, not an actual type itself, 

136 we cannot represent this with type annotations. 

137 """ 

138 return specs._make(self.addTable(name, spec) for name, spec in zip(specs._fields, specs)) 

139 

140 

141class Database(ABC): 

142 """An abstract interface that represents a particular database engine's 

143 representation of a single schema/namespace/database. 

144 

145 Parameters 

146 ---------- 

147 origin : `int` 

148 An integer ID that should be used as the default for any datasets, 

149 quanta, or other entities that use a (autoincrement, origin) compound 

150 primary key. 

151 connection : `sqlalchemy.engine.Connection` 

152 The SQLAlchemy connection this `Database` wraps. 

153 namespace : `str`, optional 

154 Name of the schema or namespace this instance is associated with. 

155 This is passed as the ``schema`` argument when constructing a 

156 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

157 avoid confusion between "schema means namespace" and "schema means 

158 table definitions". 

159 

160 Notes 

161 ----- 

162 `Database` requires all write operations to go through its special named 

163 methods. Our write patterns are sufficiently simple that we don't really 

164 need the full flexibility of SQL insert/update/delete syntax, and we need 

165 non-standard (but common) functionality in these operations sufficiently 

166 often that it seems worthwhile to provide our own generic API. 

167 

168 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

169 their SQLAlchemy representation) to be run, as we expect these to require 

170 significantly more sophistication while still being limited to standard 

171 SQL. 

172 

173 `Database` itself has several underscore-prefixed attributes: 

174 

175 - ``_cs``: SQLAlchemy objects representing the connection and transaction 

176 state. 

177 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

178 the tables and other schema entities. 

179 

180 These are considered protected (derived classes may access them, but other 

181 code should not), and read-only, aside from executing SQL via 

182 ``_connection``. 

183 """ 

184 

185 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection, 

186 namespace: Optional[str] = None): 

187 self.origin = origin 

188 self.namespace = namespace 

189 self._connection = connection 

190 self._metadata = None 

191 

192 @classmethod 

193 def makeDefaultUri(cls, root: str) -> Optional[str]: 

194 """Create a default connection URI appropriate for the given root 

195 directory, or `None` if there can be no such default. 

196 """ 

197 return None 

198 

199 @classmethod 

200 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

201 writeable: bool = True) -> Database: 

202 """Construct a database from a SQLAlchemy URI. 

203 

204 Parameters 

205 ---------- 

206 uri : `str` 

207 A SQLAlchemy URI connection string. 

208 origin : `int` 

209 An integer ID that should be used as the default for any datasets, 

210 quanta, or other entities that use a (autoincrement, origin) 

211 compound primary key. 

212 namespace : `str`, optional 

213 A database namespace (i.e. schema) the new instance should be 

214 associated with. If `None` (default), the namespace (if any) is 

215 inferred from the URI. 

216 writeable : `bool`, optional 

217 If `True`, allow write operations on the database, including 

218 ``CREATE TABLE``. 

219 

220 Returns 

221 ------- 

222 db : `Database` 

223 A new `Database` instance. 

224 """ 

225 return cls.fromConnection(cls.connect(uri, writeable=writeable), 

226 origin=origin, 

227 namespace=namespace, 

228 writeable=writeable) 

229 

230 @classmethod 

231 @abstractmethod 

232 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection: 

233 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI. 

234 

235 Parameters 

236 ---------- 

237 uri : `str` 

238 A SQLAlchemy URI connection string. 

239 origin : `int` 

240 An integer ID that should be used as the default for any datasets, 

241 quanta, or other entities that use a (autoincrement, origin) 

242 compound primary key. 

243 writeable : `bool`, optional 

244 If `True`, allow write operations on the database, including 

245 ``CREATE TABLE``. 

246 

247 Returns 

248 ------- 

249 connection : `sqlalchemy.engine.Connection` 

250 A database connection. 

251 

252 Notes 

253 ----- 

254 Subclasses that support other ways to connect to a database are 

255 encouraged to add optional arguments to their implementation of this 

256 method, as long as they maintain compatibility with the base class 

257 call signature. 

258 """ 

259 raise NotImplementedError() 

260 

261 @classmethod 

262 @abstractmethod 

263 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int, 

264 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

265 """Create a new `Database` from an existing 

266 `sqlalchemy.engine.Connection`. 

267 

268 Parameters 

269 ---------- 

270 connection : `sqllachemy.engine.Connection` 

271 The connection for the the database. May be shared between 

272 `Database` instances. 

273 origin : `int` 

274 An integer ID that should be used as the default for any datasets, 

275 quanta, or other entities that use a (autoincrement, origin) 

276 compound primary key. 

277 namespace : `str`, optional 

278 A different database namespace (i.e. schema) the new instance 

279 should be associated with. If `None` (default), the namespace 

280 (if any) is inferred from the connection. 

281 writeable : `bool`, optional 

282 If `True`, allow write operations on the database, including 

283 ``CREATE TABLE``. 

284 

285 Returns 

286 ------- 

287 db : `Database` 

288 A new `Database` instance. 

289 

290 Notes 

291 ----- 

292 This method allows different `Database` instances to share the same 

293 connection, which is desirable when they represent different namespaces 

294 can be queried together. This also ties their transaction state, 

295 however; starting a transaction in any database automatically starts 

296 on in all other databases. 

297 """ 

298 raise NotImplementedError() 

299 

300 @contextmanager 

301 def transaction(self, *, interrupting: bool = False) -> None: 

302 """Return a context manager that represents a transaction. 

303 

304 Parameters 

305 ---------- 

306 interrupting : `bool` 

307 If `True`, this transaction block needs to be able to interrupt 

308 any existing one in order to yield correct behavior. 

309 """ 

310 assert not (interrupting and self._connection.in_transaction()), ( 

311 "Logic error in transaction nesting: an operation that would " 

312 "interrupt the active transaction context has been requested." 

313 ) 

314 if self._connection.in_transaction(): 

315 trans = self._connection.begin_nested() 

316 else: 

317 # Use a regular (non-savepoint) transaction only for the outermost 

318 # context. 

319 trans = self._connection.begin() 

320 try: 

321 yield 

322 trans.commit() 

323 except BaseException: 

324 trans.rollback() 

325 raise 

326 

327 @contextmanager 

328 def declareStaticTables(self, *, create: bool) -> StaticTablesContext: 

329 """Return a context manager in which the database's static DDL schema 

330 can be declared. 

331 

332 Parameters 

333 ---------- 

334 create : `bool` 

335 If `True`, attempt to create all tables at the end of the context. 

336 If `False`, they will be assumed to already exist. 

337 

338 Returns 

339 ------- 

340 schema : `StaticTablesContext` 

341 A helper object that is used to add new tables. 

342 

343 Raises 

344 ------ 

345 ReadOnlyDatabaseError 

346 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

347 and one or more declared tables do not already exist. 

348 

349 Examples 

350 -------- 

351 Given a `Database` instance ``db``:: 

352 

353 with db.declareStaticTables(create=True) as schema: 

354 schema.addTable("table1", TableSpec(...)) 

355 schema.addTable("table2", TableSpec(...)) 

356 

357 Notes 

358 ----- 

359 A database's static DDL schema must be declared before any dynamic 

360 tables are managed via calls to `ensureTableExists` or 

361 `getExistingTable`. The order in which static schema tables are added 

362 inside the context block is unimportant; they will automatically be 

363 sorted and added in an order consistent with their foreign key 

364 relationships. 

365 """ 

366 if create and not self.isWriteable(): 

367 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

368 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

369 try: 

370 context = StaticTablesContext(self) 

371 yield context 

372 for table, foreignKey in context._foreignKeys: 

373 table.append_constraint(foreignKey) 

374 if create: 

375 if self.namespace is not None: 

376 if self.namespace not in context._inspector.get_schema_names(): 

377 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

378 # In our tables we have columns that make use of sqlalchemy 

379 # Sequence objects. There is currently a bug in sqlalchmey that 

380 # causes a deprecation warning to be thrown on a property of 

381 # the Sequence object when the repr for the sequence is 

382 # created. Here a filter is used to catch these deprecation 

383 # warnings when tables are created. 

384 with warnings.catch_warnings(): 

385 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

386 self._metadata.create_all(self._connection) 

387 except BaseException: 

388 self._metadata.drop_all(self._connection) 

389 self._metadata = None 

390 raise 

391 

392 @abstractmethod 

393 def isWriteable(self) -> bool: 

394 """Return `True` if this database can be modified by this client. 

395 """ 

396 raise NotImplementedError() 

397 

398 @abstractmethod 

399 def __str__(self) -> str: 

400 """Return a human-readable identifier for this `Database`, including 

401 any namespace or schema that identifies its names within a `Registry`. 

402 """ 

403 raise NotImplementedError() 

404 

405 def shrinkDatabaseEntityName(self, original: str) -> str: 

406 """Return a version of the given name that fits within this database 

407 engine's length limits for table, constraint, indexes, and sequence 

408 names. 

409 

410 Implementations should not assume that simple truncation is safe, 

411 because multiple long names often begin with the same prefix. 

412 

413 The default implementation simply returns the given name. 

414 

415 Parameters 

416 ---------- 

417 original : `str` 

418 The original name. 

419 

420 Returns 

421 ------- 

422 shrunk : `str` 

423 The new, possibly shortened name. 

424 """ 

425 return original 

426 

427 def expandDatabaseEntityName(self, shrunk: str) -> str: 

428 """Retrieve the original name for a database entity that was too long 

429 to fit within the database engine's limits. 

430 

431 Parameters 

432 ---------- 

433 original : `str` 

434 The original name. 

435 

436 Returns 

437 ------- 

438 shrunk : `str` 

439 The new, possibly shortened name. 

440 """ 

441 return shrunk 

442 

443 def _mangleTableName(self, name: str) -> str: 

444 """Map a logical, user-visible table name to the true table name used 

445 in the database. 

446 

447 The default implementation returns the given name unchanged. 

448 

449 Parameters 

450 ---------- 

451 name : `str` 

452 Input table name. Should not include a namespace (i.e. schema) 

453 prefix. 

454 

455 Returns 

456 ------- 

457 mangled : `str` 

458 Mangled version of the table name (still with no namespace prefix). 

459 

460 Notes 

461 ----- 

462 Reimplementations of this method must be idempotent - mangling an 

463 already-mangled name must have no effect. 

464 """ 

465 return name 

466 

467 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

468 **kwds) -> sqlalchemy.schema.Column: 

469 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

470 

471 Parameters 

472 ---------- 

473 table : `str` 

474 Name of the table this column is being added to. 

475 spec : `FieldSpec` 

476 Specification for the field to be added. 

477 metadata : `sqlalchemy.MetaData` 

478 SQLAlchemy representation of the DDL schema this field's table is 

479 being added to. 

480 **kwds 

481 Additional keyword arguments to forward to the 

482 `sqlalchemy.schema.Column` constructor. This is provided to make 

483 it easier for derived classes to delegate to ``super()`` while 

484 making only minor changes. 

485 

486 Returns 

487 ------- 

488 column : `sqlalchemy.schema.Column` 

489 SQLAlchemy representation of the field. 

490 """ 

491 args = [spec.name, spec.getSizedColumnType()] 

492 if spec.autoincrement: 

493 # Generate a sequence to use for auto incrementing for databases 

494 # that do not support it natively. This will be ignored by 

495 # sqlalchemy for databases that do support it. 

496 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

497 metadata=metadata)) 

498 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

499 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

500 comment=spec.doc, **kwds) 

501 

502 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

503 **kwds) -> sqlalchemy.schema.ForeignKeyConstraint: 

504 """Convert a `ForeignKeySpec` to a 

505 `sqlalchemy.schema.ForeignKeyConstraint`. 

506 

507 Parameters 

508 ---------- 

509 table : `str` 

510 Name of the table this foreign key is being added to. 

511 spec : `ForeignKeySpec` 

512 Specification for the foreign key to be added. 

513 metadata : `sqlalchemy.MetaData` 

514 SQLAlchemy representation of the DDL schema this constraint is 

515 being added to. 

516 **kwds 

517 Additional keyword arguments to forward to the 

518 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

519 provided to make it easier for derived classes to delegate to 

520 ``super()`` while making only minor changes. 

521 

522 Returns 

523 ------- 

524 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

525 SQLAlchemy representation of the constraint. 

526 """ 

527 name = self.shrinkDatabaseEntityName( 

528 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

529 + list(spec.target) + list(spec.source)) 

530 ) 

531 return sqlalchemy.schema.ForeignKeyConstraint( 

532 spec.source, 

533 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

534 name=name, 

535 ondelete=spec.onDelete 

536 ) 

537 

538 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

539 **kwds) -> sqlalchemy.schema.Table: 

540 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

541 

542 Parameters 

543 ---------- 

544 spec : `TableSpec` 

545 Specification for the foreign key to be added. 

546 metadata : `sqlalchemy.MetaData` 

547 SQLAlchemy representation of the DDL schema this table is being 

548 added to. 

549 **kwds 

550 Additional keyword arguments to forward to the 

551 `sqlalchemy.schema.Table` constructor. This is provided to make it 

552 easier for derived classes to delegate to ``super()`` while making 

553 only minor changes. 

554 

555 Returns 

556 ------- 

557 table : `sqlalchemy.schema.Table` 

558 SQLAlchemy representation of the table. 

559 

560 Notes 

561 ----- 

562 This method does not handle ``spec.foreignKeys`` at all, in order to 

563 avoid circular dependencies. These are added by higher-level logic in 

564 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

565 """ 

566 name = self._mangleTableName(name) 

567 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

568 args.extend( 

569 sqlalchemy.schema.UniqueConstraint( 

570 *columns, 

571 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

572 ) 

573 for columns in spec.unique if columns not in spec.indexes 

574 ) 

575 args.extend( 

576 sqlalchemy.schema.Index( 

577 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

578 *columns, 

579 unique=(columns in spec.unique) 

580 ) 

581 for columns in spec.indexes 

582 ) 

583 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

584 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

585 

586 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause: 

587 """Ensure that a table with the given name and specification exists, 

588 creating it if necessary. 

589 

590 Parameters 

591 ---------- 

592 name : `str` 

593 Name of the table (not including namespace qualifiers). 

594 spec : `TableSpec` 

595 Specification for the table. This will be used when creating the 

596 table, and *may* be used when obtaining an existing table to check 

597 for consistency, but no such check is guaranteed. 

598 

599 Returns 

600 ------- 

601 table : `sqlalchemy.schema.Table` 

602 SQLAlchemy representation of the table. 

603 

604 Raises 

605 ------ 

606 ReadOnlyDatabaseError 

607 Raised if `isWriteable` returns `False`, and the table does not 

608 already exist. 

609 DatabaseConflictError 

610 Raised if the table exists but ``spec`` is inconsistent with its 

611 definition. 

612 

613 Notes 

614 ----- 

615 This method may not be called within transactions. It may be called on 

616 read-only databases if and only if the table does in fact already 

617 exist. 

618 

619 Subclasses may override this method, but usually should not need to. 

620 """ 

621 assert not self._connection.in_transaction(), "Table creation interrupts transactions." 

622 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

623 table = self.getExistingTable(name, spec) 

624 if table is not None: 

625 return table 

626 if not self.isWriteable(): 

627 raise ReadOnlyDatabaseError( 

628 f"Table {name} does not exist, and cannot be created " 

629 f"because database {self} is read-only." 

630 ) 

631 table = self._convertTableSpec(name, spec, self._metadata) 

632 for foreignKeySpec in spec.foreignKeys: 

633 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

634 table.create(self._connection) 

635 return table 

636 

637 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

638 """Obtain an existing table with the given name and specification. 

639 

640 Parameters 

641 ---------- 

642 name : `str` 

643 Name of the table (not including namespace qualifiers). 

644 spec : `TableSpec` 

645 Specification for the table. This will be used when creating the 

646 SQLAlchemy representation of the table, and it is used to 

647 check that the actual table in the database is consistent. 

648 

649 Returns 

650 ------- 

651 table : `sqlalchemy.schema.Table` or `None` 

652 SQLAlchemy representation of the table, or `None` if it does not 

653 exist. 

654 

655 Raises 

656 ------ 

657 DatabaseConflictError 

658 Raised if the table exists but ``spec`` is inconsistent with its 

659 definition. 

660 

661 Notes 

662 ----- 

663 This method can be called within transactions and never modifies the 

664 database. 

665 

666 Subclasses may override this method, but usually should not need to. 

667 """ 

668 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

669 name = self._mangleTableName(name) 

670 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

671 if table is not None: 

672 if spec.fields.names != set(table.columns.keys()): 

673 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

674 f"specification has columns {list(spec.fields.names)}, while " 

675 f"the previous definition has {list(table.columns.keys())}.") 

676 else: 

677 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

678 if name in inspector.get_table_names(schema=self.namespace): 

679 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

680 table = self._convertTableSpec(name, spec, self._metadata) 

681 for foreignKeySpec in spec.foreignKeys: 

682 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

683 return table 

684 return table 

685 

686 def sync(self, table: sqlalchemy.schema.Table, *, 

687 keys: Dict[str, Any], 

688 compared: Optional[Dict[str, Any]] = None, 

689 extra: Optional[Dict[str, Any]] = None, 

690 returning: Optional[Sequence[str]] = None, 

691 ) -> Tuple[Optional[Dict[str, Any], bool]]: 

692 """Insert into a table as necessary to ensure database contains 

693 values equivalent to the given ones. 

694 

695 Parameters 

696 ---------- 

697 table : `sqlalchemy.schema.Table` 

698 Table to be queried and possibly inserted into. 

699 keys : `dict` 

700 Column name-value pairs used to search for an existing row; must 

701 be a combination that can be used to select a single row if one 

702 exists. If such a row does not exist, these values are used in 

703 the insert. 

704 compared : `dict`, optional 

705 Column name-value pairs that are compared to those in any existing 

706 row. If such a row does not exist, these rows are used in the 

707 insert. 

708 extra : `dict`, optional 

709 Column name-value pairs that are ignored if a matching row exists, 

710 but used in an insert if one is necessary. 

711 returning : `~collections.abc.Sequence` of `str`, optional 

712 The names of columns whose values should be returned. 

713 

714 Returns 

715 ------- 

716 row : `dict`, optional 

717 The value of the fields indicated by ``returning``, or `None` if 

718 ``returning`` is `None`. 

719 inserted : `bool` 

720 If `True`, a new row was inserted. 

721 

722 Raises 

723 ------ 

724 DatabaseConflictError 

725 Raised if the values in ``compared`` do not match the values in the 

726 database. 

727 ReadOnlyDatabaseError 

728 Raised if `isWriteable` returns `False`, and no matching record 

729 already exists. 

730 

731 Notes 

732 ----- 

733 This method may not be called within transactions. It may be called on 

734 read-only databases if and only if the matching row does in fact 

735 already exist. 

736 """ 

737 

738 def check(): 

739 """Query for a row that matches the ``key`` argument, and compare 

740 to what was given by the caller. 

741 

742 Returns 

743 ------- 

744 n : `int` 

745 Number of matching rows. ``n != 1`` is always an error, but 

746 it's a different kind of error depending on where `check` is 

747 being called. 

748 bad : `list` of `str`, or `None` 

749 The subset of the keys of ``compared`` for which the existing 

750 values did not match the given one. Once again, ``not bad`` 

751 is always an error, but a different kind on context. `None` 

752 if ``n != 1`` 

753 result : `list` or `None` 

754 Results in the database that correspond to the columns given 

755 in ``returning``, or `None` if ``returning is None``. 

756 """ 

757 toSelect = set() 

758 if compared is not None: 

759 toSelect.update(compared.keys()) 

760 if returning is not None: 

761 toSelect.update(returning) 

762 if not toSelect: 

763 # Need to select some column, even if we just want to see 

764 # how many rows we get back. 

765 toSelect.add(next(iter(keys.keys()))) 

766 selectSql = sqlalchemy.sql.select( 

767 [table.columns[k].label(k) for k in toSelect] 

768 ).select_from(table).where( 

769 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

770 ) 

771 fetched = list(self._connection.execute(selectSql).fetchall()) 

772 if len(fetched) != 1: 

773 return len(fetched), None, None 

774 existing = fetched[0] 

775 if compared is not None: 

776 

777 def safeNotEqual(a, b): 

778 if isinstance(a, astropy.time.Time): 

779 return not time_utils.times_equal(a, b) 

780 return a != b 

781 

782 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}" 

783 for k, v in compared.items() 

784 if safeNotEqual(existing[k], v)] 

785 else: 

786 inconsistencies = [] 

787 if returning is not None: 

788 toReturn = [existing[k] for k in returning] 

789 else: 

790 toReturn = None 

791 return 1, inconsistencies, toReturn 

792 

793 if self.isWriteable(): 

794 # Database is writeable. Try an insert first, but allow it to fail 

795 # (in only specific ways). 

796 row = keys.copy() 

797 if compared is not None: 

798 row.update(compared) 

799 if extra is not None: 

800 row.update(extra) 

801 insertSql = table.insert().values(row) 

802 try: 

803 with self.transaction(interrupting=True): 

804 self._connection.execute(insertSql) 

805 # Need to perform check() for this branch inside the 

806 # transaction, so we roll back an insert that didn't do 

807 # what we expected. That limits the extent to which we 

808 # can reduce duplication between this block and the other 

809 # ones that perform similar logic. 

810 n, bad, result = check() 

811 if n < 1: 

812 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.") 

813 elif n > 2: 

814 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

815 f"unique constraint for table {table.name}.") 

816 elif bad: 

817 raise RuntimeError( 

818 f"Conflict ({bad}) in sync after successful insert; this is " 

819 f"possible if the same table is being updated by a concurrent " 

820 f"process that isn't using sync, but it may also be a bug in " 

821 f"daf_butler." 

822 ) 

823 # No exceptions, so it looks like we inserted the requested row 

824 # successfully. 

825 inserted = True 

826 except sqlalchemy.exc.IntegrityError as err: 

827 # Most likely cause is that an equivalent row already exists, 

828 # but it could also be some other constraint. Query for the 

829 # row we think we matched to resolve that question. 

830 n, bad, result = check() 

831 if n < 1: 

832 # There was no matched row; insertion failed for some 

833 # completely different reason. Just re-raise the original 

834 # IntegrityError. 

835 raise 

836 elif n > 2: 

837 # There were multiple matched rows, which means we 

838 # conflicted *and* the arguments were bad to begin with. 

839 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

840 f"unique constraint for table {table.name}.") from err 

841 elif bad: 

842 # No logic bug, but data conflicted on the keys given. 

843 raise DatabaseConflictError(f"Conflict in sync for table " 

844 f"{table.name} on column(s) {bad}.") from err 

845 # The desired row is already present and consistent with what 

846 # we tried to insert. 

847 inserted = False 

848 else: 

849 assert not self._connection.in_transaction(), ( 

850 "Calling sync within a transaction block is an error even " 

851 "on a read-only database." 

852 ) 

853 # Database is not writeable; just see if the row exists. 

854 n, bad, result = check() 

855 if n < 1: 

856 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

857 elif n > 1: 

858 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

859 elif bad: 

860 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.") 

861 inserted = False 

862 return {k: v for k, v in zip(returning, result)} if returning is not None else None, inserted 

863 

864 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

865 ) -> Optional[List[int]]: 

866 """Insert one or more rows into a table, optionally returning 

867 autoincrement primary key values. 

868 

869 Parameters 

870 ---------- 

871 table : `sqlalchemy.schema.Table` 

872 Table rows should be inserted into. 

873 returnIds: `bool` 

874 If `True` (`False` is default), return the values of the table's 

875 autoincrement primary key field (which much exist). 

876 *rows 

877 Positional arguments are the rows to be inserted, as dictionaries 

878 mapping column name to value. The keys in all dictionaries must 

879 be the same. 

880 

881 Returns 

882 ------- 

883 ids : `None`, or `list` of `int` 

884 If ``returnIds`` is `True`, a `list` containing the inserted 

885 values for the table's autoincrement primary key. 

886 

887 Raises 

888 ------ 

889 ReadOnlyDatabaseError 

890 Raised if `isWriteable` returns `False` when this method is called. 

891 

892 Notes 

893 ----- 

894 The default implementation uses bulk insert syntax when ``returnIds`` 

895 is `False`, and a loop over single-row insert operations when it is 

896 `True`. 

897 

898 Derived classes should reimplement when they can provide a more 

899 efficient implementation (especially for the latter case). 

900 

901 May be used inside transaction contexts, so implementations may not 

902 perform operations that interrupt transactions. 

903 """ 

904 if not self.isWriteable(): 

905 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.") 

906 if not returnIds: 

907 self._connection.execute(table.insert(), *rows) 

908 else: 

909 sql = table.insert() 

910 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

911 

912 @abstractmethod 

913 def replace(self, table: sqlalchemy.schema.Table, *rows: dict): 

914 """Insert one or more rows into a table, replacing any existing rows 

915 for which insertion of a new row would violate the primary key 

916 constraint. 

917 

918 Parameters 

919 ---------- 

920 table : `sqlalchemy.schema.Table` 

921 Table rows should be inserted into. 

922 *rows 

923 Positional arguments are the rows to be inserted, as dictionaries 

924 mapping column name to value. The keys in all dictionaries must 

925 be the same. 

926 

927 Raises 

928 ------ 

929 ReadOnlyDatabaseError 

930 Raised if `isWriteable` returns `False` when this method is called. 

931 

932 Notes 

933 ----- 

934 May be used inside transaction contexts, so implementations may not 

935 perform operations that interrupt transactions. 

936 

937 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

938 exception when a constraint other than the primary key would be 

939 violated. 

940 

941 Implementations are not required to support `replace` on tables 

942 with autoincrement keys. 

943 """ 

944 raise NotImplementedError() 

945 

946 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

947 """Delete one or more rows from a table. 

948 

949 Parameters 

950 ---------- 

951 table : `sqlalchemy.schema.Table` 

952 Table that rows should be deleted from. 

953 columns: `~collections.abc.Iterable` of `str` 

954 The names of columns that will be used to constrain the rows to 

955 be deleted; these will be combined via ``AND`` to form the 

956 ``WHERE`` clause of the delete query. 

957 *rows 

958 Positional arguments are the keys of rows to be deleted, as 

959 dictionaries mapping column name to value. The keys in all 

960 dictionaries must exactly the names in ``columns``. 

961 

962 Returns 

963 ------- 

964 count : `int` 

965 Number of rows deleted. 

966 

967 Raises 

968 ------ 

969 ReadOnlyDatabaseError 

970 Raised if `isWriteable` returns `False` when this method is called. 

971 

972 Notes 

973 ----- 

974 May be used inside transaction contexts, so implementations may not 

975 perform operations that interrupt transactions. 

976 

977 The default implementation should be sufficient for most derived 

978 classes. 

979 """ 

980 if not self.isWriteable(): 

981 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.") 

982 sql = table.delete() 

983 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

984 if whereTerms: 

985 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

986 return self._connection.execute(sql, *rows).rowcount 

987 

988 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

989 """Update one or more rows in a table. 

990 

991 Parameters 

992 ---------- 

993 table : `sqlalchemy.schema.Table` 

994 Table containing the rows to be updated. 

995 where : `dict` [`str`, `str`] 

996 A mapping from the names of columns that will be used to search for 

997 existing rows to the keys that will hold these values in the 

998 ``rows`` dictionaries. Note that these may not be the same due to 

999 SQLAlchemy limitations. 

1000 *rows 

1001 Positional arguments are the rows to be updated. The keys in all 

1002 dictionaries must be the same, and may correspond to either a 

1003 value in the ``where`` dictionary or the name of a column to be 

1004 updated. 

1005 

1006 Returns 

1007 ------- 

1008 count : `int` 

1009 Number of rows matched (regardless of whether the update actually 

1010 modified them). 

1011 

1012 Raises 

1013 ------ 

1014 ReadOnlyDatabaseError 

1015 Raised if `isWriteable` returns `False` when this method is called. 

1016 

1017 Notes 

1018 ----- 

1019 May be used inside transaction contexts, so implementations may not 

1020 perform operations that interrupt transactions. 

1021 

1022 The default implementation should be sufficient for most derived 

1023 classes. 

1024 """ 

1025 if not self.isWriteable(): 

1026 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.") 

1027 sql = table.update().where( 

1028 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1029 ) 

1030 return self._connection.execute(sql, *rows).rowcount 

1031 

1032 def query(self, sql: sqlalchemy.sql.FromClause, *args, **kwds) -> sqlalchemy.engine.ResultProxy: 

1033 """Run a SELECT query against the database. 

1034 

1035 Parameters 

1036 ---------- 

1037 sql : `sqlalchemy.sql.FromClause` 

1038 A SQLAlchemy representation of a ``SELECT`` query. 

1039 *args 

1040 Additional positional arguments are forwarded to 

1041 `sqlalchemy.engine.Connection.execute`. 

1042 **kwds 

1043 Additional keyword arguments are forwarded to 

1044 `sqlalchemy.engine.Connection.execute`. 

1045 

1046 Returns 

1047 ------- 

1048 result : `sqlalchemy.engine.ResultProxy` 

1049 Query results. 

1050 

1051 Notes 

1052 ----- 

1053 The default implementation should be sufficient for most derived 

1054 classes. 

1055 """ 

1056 # TODO: should we guard against non-SELECT queries here? 

1057 return self._connection.execute(sql, *args, **kwds) 

1058 

1059 origin: int 

1060 """An integer ID that should be used as the default for any datasets, 

1061 quanta, or other entities that use a (autoincrement, origin) compound 

1062 primary key (`int`). 

1063 """ 

1064 

1065 namespace: Optional[str] 

1066 """The schema or namespace this database instance is associated with 

1067 (`str` or `None`). 

1068 """