Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = [ 

24 "Database", 

25 "ReadOnlyDatabaseError", 

26 "DatabaseConflictError", 

27 "StaticTablesContext", 

28] 

29 

30from abc import ABC, abstractmethod 

31from contextlib import contextmanager 

32from typing import ( 

33 Any, 

34 Callable, 

35 Dict, 

36 Iterable, 

37 Iterator, 

38 List, 

39 Optional, 

40 Sequence, 

41 Set, 

42 Tuple, 

43) 

44import warnings 

45 

46import astropy.time 

47import sqlalchemy 

48 

49from ...core import ddl, time_utils 

50from .._exceptions import ConflictingDefinitionError 

51 

52 

53def _checkExistingTableDefinition(name: str, spec: ddl.TableSpec, inspection: List[Dict[str, Any]]) -> None: 

54 """Test that the definition of a table in a `ddl.TableSpec` and from 

55 database introspection are consistent. 

56 

57 Parameters 

58 ---------- 

59 name : `str` 

60 Name of the table (only used in error messages). 

61 spec : `ddl.TableSpec` 

62 Specification of the table. 

63 inspection : `dict` 

64 Dictionary returned by 

65 `sqlalchemy.engine.reflection.Inspector.get_columns`. 

66 

67 Raises 

68 ------ 

69 DatabaseConflictError 

70 Raised if the definitions are inconsistent. 

71 """ 

72 columnNames = [c["name"] for c in inspection] 

73 if spec.fields.names != set(columnNames): 

74 raise DatabaseConflictError(f"Table '{name}' exists but is defined differently in the database; " 

75 f"specification has columns {list(spec.fields.names)}, while the " 

76 f"table in the database has {columnNames}.") 

77 

78 

79class ReadOnlyDatabaseError(RuntimeError): 

80 """Exception raised when a write operation is called on a read-only 

81 `Database`. 

82 """ 

83 

84 

85class DatabaseConflictError(ConflictingDefinitionError): 

86 """Exception raised when database content (row values or schema entities) 

87 are inconsistent with what this client expects. 

88 """ 

89 

90 

91class StaticTablesContext: 

92 """Helper class used to declare the static schema for a registry layer 

93 in a database. 

94 

95 An instance of this class is returned by `Database.declareStaticTables`, 

96 which should be the only way it should be constructed. 

97 """ 

98 

99 def __init__(self, db: Database): 

100 self._db = db 

101 self._foreignKeys: List[Tuple[sqlalchemy.schema.Table, sqlalchemy.schema.ForeignKeyConstraint]] = [] 

102 self._inspector = sqlalchemy.engine.reflection.Inspector(self._db._connection) 

103 self._tableNames = frozenset(self._inspector.get_table_names(schema=self._db.namespace)) 

104 self._initializers: List[Callable[[Database], None]] = [] 

105 

106 def addTable(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.schema.Table: 

107 """Add a new table to the schema, returning its sqlalchemy 

108 representation. 

109 

110 The new table may not actually be created until the end of the 

111 context created by `Database.declareStaticTables`, allowing tables 

112 to be declared in any order even in the presence of foreign key 

113 relationships. 

114 """ 

115 name = self._db._mangleTableName(name) 

116 if name in self._tableNames: 

117 _checkExistingTableDefinition(name, spec, self._inspector.get_columns(name, 

118 schema=self._db.namespace)) 

119 table = self._db._convertTableSpec(name, spec, self._db._metadata) 

120 for foreignKeySpec in spec.foreignKeys: 

121 self._foreignKeys.append( 

122 (table, self._db._convertForeignKeySpec(name, foreignKeySpec, self._db._metadata)) 

123 ) 

124 return table 

125 

126 def addTableTuple(self, specs: Tuple[ddl.TableSpec, ...]) -> Tuple[sqlalchemy.schema.Table, ...]: 

127 """Add a named tuple of tables to the schema, returning their 

128 SQLAlchemy representations in a named tuple of the same type. 

129 

130 The new tables may not actually be created until the end of the 

131 context created by `Database.declareStaticTables`, allowing tables 

132 to be declared in any order even in the presence of foreign key 

133 relationships. 

134 

135 Notes 

136 ----- 

137 ``specs`` *must* be an instance of a type created by 

138 `collections.namedtuple`, not just regular tuple, and the returned 

139 object is guaranteed to be the same. Because `~collections.namedtuple` 

140 is just a factory for `type` objects, not an actual type itself, 

141 we cannot represent this with type annotations. 

142 """ 

143 return specs._make(self.addTable(name, spec) # type: ignore 

144 for name, spec in zip(specs._fields, specs)) # type: ignore 

145 

146 def addInitializer(self, initializer: Callable[[Database], None]) -> None: 

147 """Add a method that does one-time initialization of a database. 

148 

149 Initialization can mean anything that changes state of a database 

150 and needs to be done exactly once after database schema was created. 

151 An example for that could be population of schema attributes. 

152 

153 Parameters 

154 ---------- 

155 initializer : callable 

156 Method of a single argument which is a `Database` instance. 

157 """ 

158 self._initializers.append(initializer) 

159 

160 

161class Database(ABC): 

162 """An abstract interface that represents a particular database engine's 

163 representation of a single schema/namespace/database. 

164 

165 Parameters 

166 ---------- 

167 origin : `int` 

168 An integer ID that should be used as the default for any datasets, 

169 quanta, or other entities that use a (autoincrement, origin) compound 

170 primary key. 

171 connection : `sqlalchemy.engine.Connection` 

172 The SQLAlchemy connection this `Database` wraps. 

173 namespace : `str`, optional 

174 Name of the schema or namespace this instance is associated with. 

175 This is passed as the ``schema`` argument when constructing a 

176 `sqlalchemy.schema.MetaData` instance. We use ``namespace`` instead to 

177 avoid confusion between "schema means namespace" and "schema means 

178 table definitions". 

179 

180 Notes 

181 ----- 

182 `Database` requires all write operations to go through its special named 

183 methods. Our write patterns are sufficiently simple that we don't really 

184 need the full flexibility of SQL insert/update/delete syntax, and we need 

185 non-standard (but common) functionality in these operations sufficiently 

186 often that it seems worthwhile to provide our own generic API. 

187 

188 In contrast, `Database.query` allows arbitrary ``SELECT`` queries (via 

189 their SQLAlchemy representation) to be run, as we expect these to require 

190 significantly more sophistication while still being limited to standard 

191 SQL. 

192 

193 `Database` itself has several underscore-prefixed attributes: 

194 

195 - ``_connection``: SQLAlchemy object representing the connection. 

196 - ``_metadata``: the `sqlalchemy.schema.MetaData` object representing 

197 the tables and other schema entities. 

198 

199 These are considered protected (derived classes may access them, but other 

200 code should not), and read-only, aside from executing SQL via 

201 ``_connection``. 

202 """ 

203 

204 def __init__(self, *, origin: int, connection: sqlalchemy.engine.Connection, 

205 namespace: Optional[str] = None): 

206 self.origin = origin 

207 self.namespace = namespace 

208 self._connection = connection 

209 self._metadata: Optional[sqlalchemy.schema.MetaData] = None 

210 

211 @classmethod 

212 def makeDefaultUri(cls, root: str) -> Optional[str]: 

213 """Create a default connection URI appropriate for the given root 

214 directory, or `None` if there can be no such default. 

215 """ 

216 return None 

217 

218 @classmethod 

219 def fromUri(cls, uri: str, *, origin: int, namespace: Optional[str] = None, 

220 writeable: bool = True) -> Database: 

221 """Construct a database from a SQLAlchemy URI. 

222 

223 Parameters 

224 ---------- 

225 uri : `str` 

226 A SQLAlchemy URI connection string. 

227 origin : `int` 

228 An integer ID that should be used as the default for any datasets, 

229 quanta, or other entities that use a (autoincrement, origin) 

230 compound primary key. 

231 namespace : `str`, optional 

232 A database namespace (i.e. schema) the new instance should be 

233 associated with. If `None` (default), the namespace (if any) is 

234 inferred from the URI. 

235 writeable : `bool`, optional 

236 If `True`, allow write operations on the database, including 

237 ``CREATE TABLE``. 

238 

239 Returns 

240 ------- 

241 db : `Database` 

242 A new `Database` instance. 

243 """ 

244 return cls.fromConnection(cls.connect(uri, writeable=writeable), 

245 origin=origin, 

246 namespace=namespace, 

247 writeable=writeable) 

248 

249 @classmethod 

250 @abstractmethod 

251 def connect(cls, uri: str, *, writeable: bool = True) -> sqlalchemy.engine.Connection: 

252 """Create a `sqlalchemy.engine.Connection` from a SQLAlchemy URI. 

253 

254 Parameters 

255 ---------- 

256 uri : `str` 

257 A SQLAlchemy URI connection string. 

258 origin : `int` 

259 An integer ID that should be used as the default for any datasets, 

260 quanta, or other entities that use a (autoincrement, origin) 

261 compound primary key. 

262 writeable : `bool`, optional 

263 If `True`, allow write operations on the database, including 

264 ``CREATE TABLE``. 

265 

266 Returns 

267 ------- 

268 connection : `sqlalchemy.engine.Connection` 

269 A database connection. 

270 

271 Notes 

272 ----- 

273 Subclasses that support other ways to connect to a database are 

274 encouraged to add optional arguments to their implementation of this 

275 method, as long as they maintain compatibility with the base class 

276 call signature. 

277 """ 

278 raise NotImplementedError() 

279 

280 @classmethod 

281 @abstractmethod 

282 def fromConnection(cls, connection: sqlalchemy.engine.Connection, *, origin: int, 

283 namespace: Optional[str] = None, writeable: bool = True) -> Database: 

284 """Create a new `Database` from an existing 

285 `sqlalchemy.engine.Connection`. 

286 

287 Parameters 

288 ---------- 

289 connection : `sqllachemy.engine.Connection` 

290 The connection for the the database. May be shared between 

291 `Database` instances. 

292 origin : `int` 

293 An integer ID that should be used as the default for any datasets, 

294 quanta, or other entities that use a (autoincrement, origin) 

295 compound primary key. 

296 namespace : `str`, optional 

297 A different database namespace (i.e. schema) the new instance 

298 should be associated with. If `None` (default), the namespace 

299 (if any) is inferred from the connection. 

300 writeable : `bool`, optional 

301 If `True`, allow write operations on the database, including 

302 ``CREATE TABLE``. 

303 

304 Returns 

305 ------- 

306 db : `Database` 

307 A new `Database` instance. 

308 

309 Notes 

310 ----- 

311 This method allows different `Database` instances to share the same 

312 connection, which is desirable when they represent different namespaces 

313 can be queried together. This also ties their transaction state, 

314 however; starting a transaction in any database automatically starts 

315 on in all other databases. 

316 """ 

317 raise NotImplementedError() 

318 

319 @contextmanager 

320 def transaction(self, *, interrupting: bool = False) -> Iterator: 

321 """Return a context manager that represents a transaction. 

322 

323 Parameters 

324 ---------- 

325 interrupting : `bool` 

326 If `True`, this transaction block needs to be able to interrupt 

327 any existing one in order to yield correct behavior. 

328 """ 

329 assert not (interrupting and self._connection.in_transaction()), ( 

330 "Logic error in transaction nesting: an operation that would " 

331 "interrupt the active transaction context has been requested." 

332 ) 

333 if self._connection.in_transaction(): 

334 trans = self._connection.begin_nested() 

335 else: 

336 # Use a regular (non-savepoint) transaction only for the outermost 

337 # context. 

338 trans = self._connection.begin() 

339 try: 

340 yield 

341 trans.commit() 

342 except BaseException: 

343 trans.rollback() 

344 raise 

345 

346 @contextmanager 

347 def declareStaticTables(self, *, create: bool) -> Iterator[StaticTablesContext]: 

348 """Return a context manager in which the database's static DDL schema 

349 can be declared. 

350 

351 Parameters 

352 ---------- 

353 create : `bool` 

354 If `True`, attempt to create all tables at the end of the context. 

355 If `False`, they will be assumed to already exist. 

356 

357 Returns 

358 ------- 

359 schema : `StaticTablesContext` 

360 A helper object that is used to add new tables. 

361 

362 Raises 

363 ------ 

364 ReadOnlyDatabaseError 

365 Raised if ``create`` is `True`, `Database.isWriteable` is `False`, 

366 and one or more declared tables do not already exist. 

367 

368 Examples 

369 -------- 

370 Given a `Database` instance ``db``:: 

371 

372 with db.declareStaticTables(create=True) as schema: 

373 schema.addTable("table1", TableSpec(...)) 

374 schema.addTable("table2", TableSpec(...)) 

375 

376 Notes 

377 ----- 

378 A database's static DDL schema must be declared before any dynamic 

379 tables are managed via calls to `ensureTableExists` or 

380 `getExistingTable`. The order in which static schema tables are added 

381 inside the context block is unimportant; they will automatically be 

382 sorted and added in an order consistent with their foreign key 

383 relationships. 

384 """ 

385 if create and not self.isWriteable(): 

386 raise ReadOnlyDatabaseError(f"Cannot create tables in read-only database {self}.") 

387 self._metadata = sqlalchemy.MetaData(schema=self.namespace) 

388 try: 

389 context = StaticTablesContext(self) 

390 yield context 

391 for table, foreignKey in context._foreignKeys: 

392 table.append_constraint(foreignKey) 

393 if create: 

394 if self.namespace is not None: 

395 if self.namespace not in context._inspector.get_schema_names(): 

396 self._connection.execute(sqlalchemy.schema.CreateSchema(self.namespace)) 

397 # In our tables we have columns that make use of sqlalchemy 

398 # Sequence objects. There is currently a bug in sqlalchmey that 

399 # causes a deprecation warning to be thrown on a property of 

400 # the Sequence object when the repr for the sequence is 

401 # created. Here a filter is used to catch these deprecation 

402 # warnings when tables are created. 

403 with warnings.catch_warnings(): 

404 warnings.simplefilter("ignore", category=sqlalchemy.exc.SADeprecationWarning) 

405 self._metadata.create_all(self._connection) 

406 # call all initializer methods sequentially 

407 for init in context._initializers: 

408 init(self) 

409 except BaseException: 

410 # TODO: this is potentially dangerous if we run it on 

411 # pre-existing schema. 

412 self._metadata.drop_all(self._connection) 

413 self._metadata = None 

414 raise 

415 

416 @abstractmethod 

417 def isWriteable(self) -> bool: 

418 """Return `True` if this database can be modified by this client. 

419 """ 

420 raise NotImplementedError() 

421 

422 @abstractmethod 

423 def __str__(self) -> str: 

424 """Return a human-readable identifier for this `Database`, including 

425 any namespace or schema that identifies its names within a `Registry`. 

426 """ 

427 raise NotImplementedError() 

428 

429 def shrinkDatabaseEntityName(self, original: str) -> str: 

430 """Return a version of the given name that fits within this database 

431 engine's length limits for table, constraint, indexes, and sequence 

432 names. 

433 

434 Implementations should not assume that simple truncation is safe, 

435 because multiple long names often begin with the same prefix. 

436 

437 The default implementation simply returns the given name. 

438 

439 Parameters 

440 ---------- 

441 original : `str` 

442 The original name. 

443 

444 Returns 

445 ------- 

446 shrunk : `str` 

447 The new, possibly shortened name. 

448 """ 

449 return original 

450 

451 def expandDatabaseEntityName(self, shrunk: str) -> str: 

452 """Retrieve the original name for a database entity that was too long 

453 to fit within the database engine's limits. 

454 

455 Parameters 

456 ---------- 

457 original : `str` 

458 The original name. 

459 

460 Returns 

461 ------- 

462 shrunk : `str` 

463 The new, possibly shortened name. 

464 """ 

465 return shrunk 

466 

467 def _mangleTableName(self, name: str) -> str: 

468 """Map a logical, user-visible table name to the true table name used 

469 in the database. 

470 

471 The default implementation returns the given name unchanged. 

472 

473 Parameters 

474 ---------- 

475 name : `str` 

476 Input table name. Should not include a namespace (i.e. schema) 

477 prefix. 

478 

479 Returns 

480 ------- 

481 mangled : `str` 

482 Mangled version of the table name (still with no namespace prefix). 

483 

484 Notes 

485 ----- 

486 Reimplementations of this method must be idempotent - mangling an 

487 already-mangled name must have no effect. 

488 """ 

489 return name 

490 

491 def _makeColumnConstraints(self, table: str, spec: ddl.FieldSpec) -> List[sqlalchemy.CheckConstraint]: 

492 """Create constraints based on this spec. 

493 

494 Parameters 

495 ---------- 

496 table : `str` 

497 Name of the table this column is being added to. 

498 spec : `FieldSpec` 

499 Specification for the field to be added. 

500 

501 Returns 

502 ------- 

503 constraint : `list` of `sqlalchemy.CheckConstraint` 

504 Constraint added for this column. 

505 """ 

506 # By default we return no additional constraints 

507 return [] 

508 

509 def _convertFieldSpec(self, table: str, spec: ddl.FieldSpec, metadata: sqlalchemy.MetaData, 

510 **kwds: Any) -> sqlalchemy.schema.Column: 

511 """Convert a `FieldSpec` to a `sqlalchemy.schema.Column`. 

512 

513 Parameters 

514 ---------- 

515 table : `str` 

516 Name of the table this column is being added to. 

517 spec : `FieldSpec` 

518 Specification for the field to be added. 

519 metadata : `sqlalchemy.MetaData` 

520 SQLAlchemy representation of the DDL schema this field's table is 

521 being added to. 

522 **kwds 

523 Additional keyword arguments to forward to the 

524 `sqlalchemy.schema.Column` constructor. This is provided to make 

525 it easier for derived classes to delegate to ``super()`` while 

526 making only minor changes. 

527 

528 Returns 

529 ------- 

530 column : `sqlalchemy.schema.Column` 

531 SQLAlchemy representation of the field. 

532 """ 

533 args = [spec.name, spec.getSizedColumnType()] 

534 if spec.autoincrement: 

535 # Generate a sequence to use for auto incrementing for databases 

536 # that do not support it natively. This will be ignored by 

537 # sqlalchemy for databases that do support it. 

538 args.append(sqlalchemy.Sequence(self.shrinkDatabaseEntityName(f"{table}_seq_{spec.name}"), 

539 metadata=metadata)) 

540 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {table}.{spec.name}." 

541 return sqlalchemy.schema.Column(*args, nullable=spec.nullable, primary_key=spec.primaryKey, 

542 comment=spec.doc, **kwds) 

543 

544 def _convertForeignKeySpec(self, table: str, spec: ddl.ForeignKeySpec, metadata: sqlalchemy.MetaData, 

545 **kwds: Any) -> sqlalchemy.schema.ForeignKeyConstraint: 

546 """Convert a `ForeignKeySpec` to a 

547 `sqlalchemy.schema.ForeignKeyConstraint`. 

548 

549 Parameters 

550 ---------- 

551 table : `str` 

552 Name of the table this foreign key is being added to. 

553 spec : `ForeignKeySpec` 

554 Specification for the foreign key to be added. 

555 metadata : `sqlalchemy.MetaData` 

556 SQLAlchemy representation of the DDL schema this constraint is 

557 being added to. 

558 **kwds 

559 Additional keyword arguments to forward to the 

560 `sqlalchemy.schema.ForeignKeyConstraint` constructor. This is 

561 provided to make it easier for derived classes to delegate to 

562 ``super()`` while making only minor changes. 

563 

564 Returns 

565 ------- 

566 constraint : `sqlalchemy.schema.ForeignKeyConstraint` 

567 SQLAlchemy representation of the constraint. 

568 """ 

569 name = self.shrinkDatabaseEntityName( 

570 "_".join(["fkey", table, self._mangleTableName(spec.table)] 

571 + list(spec.target) + list(spec.source)) 

572 ) 

573 return sqlalchemy.schema.ForeignKeyConstraint( 

574 spec.source, 

575 [f"{self._mangleTableName(spec.table)}.{col}" for col in spec.target], 

576 name=name, 

577 ondelete=spec.onDelete 

578 ) 

579 

580 def _convertTableSpec(self, name: str, spec: ddl.TableSpec, metadata: sqlalchemy.MetaData, 

581 **kwds: Any) -> sqlalchemy.schema.Table: 

582 """Convert a `TableSpec` to a `sqlalchemy.schema.Table`. 

583 

584 Parameters 

585 ---------- 

586 spec : `TableSpec` 

587 Specification for the foreign key to be added. 

588 metadata : `sqlalchemy.MetaData` 

589 SQLAlchemy representation of the DDL schema this table is being 

590 added to. 

591 **kwds 

592 Additional keyword arguments to forward to the 

593 `sqlalchemy.schema.Table` constructor. This is provided to make it 

594 easier for derived classes to delegate to ``super()`` while making 

595 only minor changes. 

596 

597 Returns 

598 ------- 

599 table : `sqlalchemy.schema.Table` 

600 SQLAlchemy representation of the table. 

601 

602 Notes 

603 ----- 

604 This method does not handle ``spec.foreignKeys`` at all, in order to 

605 avoid circular dependencies. These are added by higher-level logic in 

606 `ensureTableExists`, `getExistingTable`, and `declareStaticTables`. 

607 """ 

608 name = self._mangleTableName(name) 

609 args = [self._convertFieldSpec(name, fieldSpec, metadata) for fieldSpec in spec.fields] 

610 

611 # Add any column constraints 

612 for fieldSpec in spec.fields: 

613 args.extend(self._makeColumnConstraints(name, fieldSpec)) 

614 

615 # Track indexes added for primary key and unique constraints, to make 

616 # sure we don't add duplicate explicit or foreign key indexes for 

617 # those. 

618 allIndexes = {tuple(fieldSpec.name for fieldSpec in spec.fields if fieldSpec.primaryKey)} 

619 args.extend( 

620 sqlalchemy.schema.UniqueConstraint( 

621 *columns, 

622 name=self.shrinkDatabaseEntityName("_".join([name, "unq"] + list(columns))) 

623 ) 

624 for columns in spec.unique 

625 ) 

626 allIndexes.update(spec.unique) 

627 args.extend( 

628 sqlalchemy.schema.Index( 

629 self.shrinkDatabaseEntityName("_".join([name, "idx"] + list(columns))), 

630 *columns, 

631 unique=(columns in spec.unique) 

632 ) 

633 for columns in spec.indexes if columns not in allIndexes 

634 ) 

635 allIndexes.update(spec.indexes) 

636 args.extend( 

637 sqlalchemy.schema.Index( 

638 self.shrinkDatabaseEntityName("_".join((name, "fkidx") + fk.source)), 

639 *fk.source, 

640 ) 

641 for fk in spec.foreignKeys if fk.addIndex and fk.source not in allIndexes 

642 ) 

643 assert spec.doc is None or isinstance(spec.doc, str), f"Bad doc for {name}." 

644 return sqlalchemy.schema.Table(name, metadata, *args, comment=spec.doc, info=spec, **kwds) 

645 

646 def ensureTableExists(self, name: str, spec: ddl.TableSpec) -> sqlalchemy.sql.FromClause: 

647 """Ensure that a table with the given name and specification exists, 

648 creating it if necessary. 

649 

650 Parameters 

651 ---------- 

652 name : `str` 

653 Name of the table (not including namespace qualifiers). 

654 spec : `TableSpec` 

655 Specification for the table. This will be used when creating the 

656 table, and *may* be used when obtaining an existing table to check 

657 for consistency, but no such check is guaranteed. 

658 

659 Returns 

660 ------- 

661 table : `sqlalchemy.schema.Table` 

662 SQLAlchemy representation of the table. 

663 

664 Raises 

665 ------ 

666 ReadOnlyDatabaseError 

667 Raised if `isWriteable` returns `False`, and the table does not 

668 already exist. 

669 DatabaseConflictError 

670 Raised if the table exists but ``spec`` is inconsistent with its 

671 definition. 

672 

673 Notes 

674 ----- 

675 This method may not be called within transactions. It may be called on 

676 read-only databases if and only if the table does in fact already 

677 exist. 

678 

679 Subclasses may override this method, but usually should not need to. 

680 """ 

681 assert not self._connection.in_transaction(), "Table creation interrupts transactions." 

682 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

683 table = self.getExistingTable(name, spec) 

684 if table is not None: 

685 return table 

686 if not self.isWriteable(): 

687 raise ReadOnlyDatabaseError( 

688 f"Table {name} does not exist, and cannot be created " 

689 f"because database {self} is read-only." 

690 ) 

691 table = self._convertTableSpec(name, spec, self._metadata) 

692 for foreignKeySpec in spec.foreignKeys: 

693 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

694 table.create(self._connection) 

695 return table 

696 

697 def getExistingTable(self, name: str, spec: ddl.TableSpec) -> Optional[sqlalchemy.schema.Table]: 

698 """Obtain an existing table with the given name and specification. 

699 

700 Parameters 

701 ---------- 

702 name : `str` 

703 Name of the table (not including namespace qualifiers). 

704 spec : `TableSpec` 

705 Specification for the table. This will be used when creating the 

706 SQLAlchemy representation of the table, and it is used to 

707 check that the actual table in the database is consistent. 

708 

709 Returns 

710 ------- 

711 table : `sqlalchemy.schema.Table` or `None` 

712 SQLAlchemy representation of the table, or `None` if it does not 

713 exist. 

714 

715 Raises 

716 ------ 

717 DatabaseConflictError 

718 Raised if the table exists but ``spec`` is inconsistent with its 

719 definition. 

720 

721 Notes 

722 ----- 

723 This method can be called within transactions and never modifies the 

724 database. 

725 

726 Subclasses may override this method, but usually should not need to. 

727 """ 

728 assert self._metadata is not None, "Static tables must be declared before dynamic tables." 

729 name = self._mangleTableName(name) 

730 table = self._metadata.tables.get(name if self.namespace is None else f"{self.namespace}.{name}") 

731 if table is not None: 

732 if spec.fields.names != set(table.columns.keys()): 

733 raise DatabaseConflictError(f"Table '{name}' has already been defined differently; the new " 

734 f"specification has columns {list(spec.fields.names)}, while " 

735 f"the previous definition has {list(table.columns.keys())}.") 

736 else: 

737 inspector = sqlalchemy.engine.reflection.Inspector(self._connection) 

738 if name in inspector.get_table_names(schema=self.namespace): 

739 _checkExistingTableDefinition(name, spec, inspector.get_columns(name, schema=self.namespace)) 

740 table = self._convertTableSpec(name, spec, self._metadata) 

741 for foreignKeySpec in spec.foreignKeys: 

742 table.append_constraint(self._convertForeignKeySpec(name, foreignKeySpec, self._metadata)) 

743 return table 

744 return table 

745 

746 def sync(self, table: sqlalchemy.schema.Table, *, 

747 keys: Dict[str, Any], 

748 compared: Optional[Dict[str, Any]] = None, 

749 extra: Optional[Dict[str, Any]] = None, 

750 returning: Optional[Sequence[str]] = None, 

751 ) -> Tuple[Optional[Dict[str, Any]], bool]: 

752 """Insert into a table as necessary to ensure database contains 

753 values equivalent to the given ones. 

754 

755 Parameters 

756 ---------- 

757 table : `sqlalchemy.schema.Table` 

758 Table to be queried and possibly inserted into. 

759 keys : `dict` 

760 Column name-value pairs used to search for an existing row; must 

761 be a combination that can be used to select a single row if one 

762 exists. If such a row does not exist, these values are used in 

763 the insert. 

764 compared : `dict`, optional 

765 Column name-value pairs that are compared to those in any existing 

766 row. If such a row does not exist, these rows are used in the 

767 insert. 

768 extra : `dict`, optional 

769 Column name-value pairs that are ignored if a matching row exists, 

770 but used in an insert if one is necessary. 

771 returning : `~collections.abc.Sequence` of `str`, optional 

772 The names of columns whose values should be returned. 

773 

774 Returns 

775 ------- 

776 row : `dict`, optional 

777 The value of the fields indicated by ``returning``, or `None` if 

778 ``returning`` is `None`. 

779 inserted : `bool` 

780 If `True`, a new row was inserted. 

781 

782 Raises 

783 ------ 

784 DatabaseConflictError 

785 Raised if the values in ``compared`` do not match the values in the 

786 database. 

787 ReadOnlyDatabaseError 

788 Raised if `isWriteable` returns `False`, and no matching record 

789 already exists. 

790 

791 Notes 

792 ----- 

793 This method may not be called within transactions. It may be called on 

794 read-only databases if and only if the matching row does in fact 

795 already exist. 

796 """ 

797 

798 def check() -> Tuple[int, Optional[List[str]], Optional[List]]: 

799 """Query for a row that matches the ``key`` argument, and compare 

800 to what was given by the caller. 

801 

802 Returns 

803 ------- 

804 n : `int` 

805 Number of matching rows. ``n != 1`` is always an error, but 

806 it's a different kind of error depending on where `check` is 

807 being called. 

808 bad : `list` of `str`, or `None` 

809 The subset of the keys of ``compared`` for which the existing 

810 values did not match the given one. Once again, ``not bad`` 

811 is always an error, but a different kind on context. `None` 

812 if ``n != 1`` 

813 result : `list` or `None` 

814 Results in the database that correspond to the columns given 

815 in ``returning``, or `None` if ``returning is None``. 

816 """ 

817 toSelect: Set[str] = set() 

818 if compared is not None: 

819 toSelect.update(compared.keys()) 

820 if returning is not None: 

821 toSelect.update(returning) 

822 if not toSelect: 

823 # Need to select some column, even if we just want to see 

824 # how many rows we get back. 

825 toSelect.add(next(iter(keys.keys()))) 

826 selectSql = sqlalchemy.sql.select( 

827 [table.columns[k].label(k) for k in toSelect] 

828 ).select_from(table).where( 

829 sqlalchemy.sql.and_(*[table.columns[k] == v for k, v in keys.items()]) 

830 ) 

831 fetched = list(self._connection.execute(selectSql).fetchall()) 

832 if len(fetched) != 1: 

833 return len(fetched), None, None 

834 existing = fetched[0] 

835 if compared is not None: 

836 

837 def safeNotEqual(a: Any, b: Any) -> bool: 

838 if isinstance(a, astropy.time.Time): 

839 return not time_utils.times_equal(a, b) 

840 return a != b 

841 

842 inconsistencies = [f"{k}: {existing[k]!r} != {v!r}" 

843 for k, v in compared.items() 

844 if safeNotEqual(existing[k], v)] 

845 else: 

846 inconsistencies = [] 

847 if returning is not None: 

848 toReturn: Optional[list] = [existing[k] for k in returning] 

849 else: 

850 toReturn = None 

851 return 1, inconsistencies, toReturn 

852 

853 if self.isWriteable(): 

854 # Database is writeable. Try an insert first, but allow it to fail 

855 # (in only specific ways). 

856 row = keys.copy() 

857 if compared is not None: 

858 row.update(compared) 

859 if extra is not None: 

860 row.update(extra) 

861 insertSql = table.insert().values(row) 

862 try: 

863 with self.transaction(interrupting=True): 

864 self._connection.execute(insertSql) 

865 # Need to perform check() for this branch inside the 

866 # transaction, so we roll back an insert that didn't do 

867 # what we expected. That limits the extent to which we 

868 # can reduce duplication between this block and the other 

869 # ones that perform similar logic. 

870 n, bad, result = check() 

871 if n < 1: 

872 raise RuntimeError("Insertion in sync did not seem to affect table. This is a bug.") 

873 elif n > 1: 

874 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

875 f"unique constraint for table {table.name}.") 

876 elif bad: 

877 raise RuntimeError( 

878 f"Conflict ({bad}) in sync after successful insert; this is " 

879 f"possible if the same table is being updated by a concurrent " 

880 f"process that isn't using sync, but it may also be a bug in " 

881 f"daf_butler." 

882 ) 

883 # No exceptions, so it looks like we inserted the requested row 

884 # successfully. 

885 inserted = True 

886 except sqlalchemy.exc.IntegrityError as err: 

887 # Most likely cause is that an equivalent row already exists, 

888 # but it could also be some other constraint. Query for the 

889 # row we think we matched to resolve that question. 

890 n, bad, result = check() 

891 if n < 1: 

892 # There was no matched row; insertion failed for some 

893 # completely different reason. Just re-raise the original 

894 # IntegrityError. 

895 raise 

896 elif n > 2: 

897 # There were multiple matched rows, which means we 

898 # conflicted *and* the arguments were bad to begin with. 

899 raise RuntimeError(f"Keys passed to sync {keys.keys()} do not comprise a " 

900 f"unique constraint for table {table.name}.") from err 

901 elif bad: 

902 # No logic bug, but data conflicted on the keys given. 

903 raise DatabaseConflictError(f"Conflict in sync for table " 

904 f"{table.name} on column(s) {bad}.") from err 

905 # The desired row is already present and consistent with what 

906 # we tried to insert. 

907 inserted = False 

908 else: 

909 assert not self._connection.in_transaction(), ( 

910 "Calling sync within a transaction block is an error even " 

911 "on a read-only database." 

912 ) 

913 # Database is not writeable; just see if the row exists. 

914 n, bad, result = check() 

915 if n < 1: 

916 raise ReadOnlyDatabaseError("sync needs to insert, but database is read-only.") 

917 elif n > 1: 

918 raise RuntimeError("Keys passed to sync do not comprise a unique constraint.") 

919 elif bad: 

920 raise DatabaseConflictError(f"Conflict in sync on column(s) {bad}.") 

921 inserted = False 

922 if returning is None: 

923 return None, inserted 

924 else: 

925 assert result is not None 

926 return {k: v for k, v in zip(returning, result)}, inserted 

927 

928 def insert(self, table: sqlalchemy.schema.Table, *rows: dict, returnIds: bool = False, 

929 ) -> Optional[List[int]]: 

930 """Insert one or more rows into a table, optionally returning 

931 autoincrement primary key values. 

932 

933 Parameters 

934 ---------- 

935 table : `sqlalchemy.schema.Table` 

936 Table rows should be inserted into. 

937 returnIds: `bool` 

938 If `True` (`False` is default), return the values of the table's 

939 autoincrement primary key field (which much exist). 

940 *rows 

941 Positional arguments are the rows to be inserted, as dictionaries 

942 mapping column name to value. The keys in all dictionaries must 

943 be the same. 

944 

945 Returns 

946 ------- 

947 ids : `None`, or `list` of `int` 

948 If ``returnIds`` is `True`, a `list` containing the inserted 

949 values for the table's autoincrement primary key. 

950 

951 Raises 

952 ------ 

953 ReadOnlyDatabaseError 

954 Raised if `isWriteable` returns `False` when this method is called. 

955 

956 Notes 

957 ----- 

958 The default implementation uses bulk insert syntax when ``returnIds`` 

959 is `False`, and a loop over single-row insert operations when it is 

960 `True`. 

961 

962 Derived classes should reimplement when they can provide a more 

963 efficient implementation (especially for the latter case). 

964 

965 May be used inside transaction contexts, so implementations may not 

966 perform operations that interrupt transactions. 

967 """ 

968 if not self.isWriteable(): 

969 raise ReadOnlyDatabaseError(f"Attempt to insert into read-only database '{self}'.") 

970 if not rows: 

971 if returnIds: 

972 return [] 

973 else: 

974 return None 

975 if not returnIds: 

976 self._connection.execute(table.insert(), *rows) 

977 return None 

978 else: 

979 sql = table.insert() 

980 return [self._connection.execute(sql, row).inserted_primary_key[0] for row in rows] 

981 

982 @abstractmethod 

983 def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None: 

984 """Insert one or more rows into a table, replacing any existing rows 

985 for which insertion of a new row would violate the primary key 

986 constraint. 

987 

988 Parameters 

989 ---------- 

990 table : `sqlalchemy.schema.Table` 

991 Table rows should be inserted into. 

992 *rows 

993 Positional arguments are the rows to be inserted, as dictionaries 

994 mapping column name to value. The keys in all dictionaries must 

995 be the same. 

996 

997 Raises 

998 ------ 

999 ReadOnlyDatabaseError 

1000 Raised if `isWriteable` returns `False` when this method is called. 

1001 

1002 Notes 

1003 ----- 

1004 May be used inside transaction contexts, so implementations may not 

1005 perform operations that interrupt transactions. 

1006 

1007 Implementations should raise a `sqlalchemy.exc.IntegrityError` 

1008 exception when a constraint other than the primary key would be 

1009 violated. 

1010 

1011 Implementations are not required to support `replace` on tables 

1012 with autoincrement keys. 

1013 """ 

1014 raise NotImplementedError() 

1015 

1016 def delete(self, table: sqlalchemy.schema.Table, columns: Iterable[str], *rows: dict) -> int: 

1017 """Delete one or more rows from a table. 

1018 

1019 Parameters 

1020 ---------- 

1021 table : `sqlalchemy.schema.Table` 

1022 Table that rows should be deleted from. 

1023 columns: `~collections.abc.Iterable` of `str` 

1024 The names of columns that will be used to constrain the rows to 

1025 be deleted; these will be combined via ``AND`` to form the 

1026 ``WHERE`` clause of the delete query. 

1027 *rows 

1028 Positional arguments are the keys of rows to be deleted, as 

1029 dictionaries mapping column name to value. The keys in all 

1030 dictionaries must exactly the names in ``columns``. 

1031 

1032 Returns 

1033 ------- 

1034 count : `int` 

1035 Number of rows deleted. 

1036 

1037 Raises 

1038 ------ 

1039 ReadOnlyDatabaseError 

1040 Raised if `isWriteable` returns `False` when this method is called. 

1041 

1042 Notes 

1043 ----- 

1044 May be used inside transaction contexts, so implementations may not 

1045 perform operations that interrupt transactions. 

1046 

1047 The default implementation should be sufficient for most derived 

1048 classes. 

1049 """ 

1050 if not self.isWriteable(): 

1051 raise ReadOnlyDatabaseError(f"Attempt to delete from read-only database '{self}'.") 

1052 if columns and not rows: 

1053 # If there are no columns, this operation is supposed to delete 

1054 # everything (so we proceed as usual). But if there are columns, 

1055 # but no rows, it was a constrained bulk operation where the 

1056 # constraint is that no rows match, and we should short-circuit 

1057 # while reporting that no rows were affected. 

1058 return 0 

1059 sql = table.delete() 

1060 whereTerms = [table.columns[name] == sqlalchemy.sql.bindparam(name) for name in columns] 

1061 if whereTerms: 

1062 sql = sql.where(sqlalchemy.sql.and_(*whereTerms)) 

1063 return self._connection.execute(sql, *rows).rowcount 

1064 

1065 def update(self, table: sqlalchemy.schema.Table, where: Dict[str, str], *rows: dict) -> int: 

1066 """Update one or more rows in a table. 

1067 

1068 Parameters 

1069 ---------- 

1070 table : `sqlalchemy.schema.Table` 

1071 Table containing the rows to be updated. 

1072 where : `dict` [`str`, `str`] 

1073 A mapping from the names of columns that will be used to search for 

1074 existing rows to the keys that will hold these values in the 

1075 ``rows`` dictionaries. Note that these may not be the same due to 

1076 SQLAlchemy limitations. 

1077 *rows 

1078 Positional arguments are the rows to be updated. The keys in all 

1079 dictionaries must be the same, and may correspond to either a 

1080 value in the ``where`` dictionary or the name of a column to be 

1081 updated. 

1082 

1083 Returns 

1084 ------- 

1085 count : `int` 

1086 Number of rows matched (regardless of whether the update actually 

1087 modified them). 

1088 

1089 Raises 

1090 ------ 

1091 ReadOnlyDatabaseError 

1092 Raised if `isWriteable` returns `False` when this method is called. 

1093 

1094 Notes 

1095 ----- 

1096 May be used inside transaction contexts, so implementations may not 

1097 perform operations that interrupt transactions. 

1098 

1099 The default implementation should be sufficient for most derived 

1100 classes. 

1101 """ 

1102 if not self.isWriteable(): 

1103 raise ReadOnlyDatabaseError(f"Attempt to update read-only database '{self}'.") 

1104 if not rows: 

1105 return 0 

1106 sql = table.update().where( 

1107 sqlalchemy.sql.and_(*[table.columns[k] == sqlalchemy.sql.bindparam(v) for k, v in where.items()]) 

1108 ) 

1109 return self._connection.execute(sql, *rows).rowcount 

1110 

1111 def query(self, sql: sqlalchemy.sql.FromClause, 

1112 *args: Any, **kwds: Any) -> sqlalchemy.engine.ResultProxy: 

1113 """Run a SELECT query against the database. 

1114 

1115 Parameters 

1116 ---------- 

1117 sql : `sqlalchemy.sql.FromClause` 

1118 A SQLAlchemy representation of a ``SELECT`` query. 

1119 *args 

1120 Additional positional arguments are forwarded to 

1121 `sqlalchemy.engine.Connection.execute`. 

1122 **kwds 

1123 Additional keyword arguments are forwarded to 

1124 `sqlalchemy.engine.Connection.execute`. 

1125 

1126 Returns 

1127 ------- 

1128 result : `sqlalchemy.engine.ResultProxy` 

1129 Query results. 

1130 

1131 Notes 

1132 ----- 

1133 The default implementation should be sufficient for most derived 

1134 classes. 

1135 """ 

1136 # TODO: should we guard against non-SELECT queries here? 

1137 return self._connection.execute(sql, *args, **kwds) 

1138 

1139 origin: int 

1140 """An integer ID that should be used as the default for any datasets, 

1141 quanta, or other entities that use a (autoincrement, origin) compound 

1142 primary key (`int`). 

1143 """ 

1144 

1145 namespace: Optional[str] 

1146 """The schema or namespace this database instance is associated with 

1147 (`str` or `None`). 

1148 """