Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "DatabaseDimensionOverlapStorage", 

25 "DatabaseDimensionRecordStorage", 

26 "DimensionRecordStorage", 

27 "DimensionRecordStorageManager", 

28 "GovernorDimensionRecordStorage", 

29 "SkyPixDimensionRecordStorage", 

30) 

31 

32from abc import ABC, abstractmethod 

33from typing import ( 

34 AbstractSet, Any, 

35 Callable, 

36 Iterable, Mapping, 

37 Optional, 

38 Tuple, 

39 TYPE_CHECKING, 

40 Union, 

41) 

42 

43import sqlalchemy 

44 

45from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension 

46from ._versioning import VersionedExtension 

47 

48if TYPE_CHECKING: 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true

49 from ...core import ( 

50 DataCoordinateIterable, 

51 DimensionElement, 

52 DimensionRecord, 

53 DimensionUniverse, 

54 NamedKeyDict, 

55 NamedKeyMapping, 

56 TimespanDatabaseRepresentation, 

57 ) 

58 from ..queries import QueryBuilder 

59 from ._database import Database, StaticTablesContext 

60 

61 

62OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]] 

63 

64 

65class DimensionRecordStorage(ABC): 

66 """An abstract base class that represents a way of storing the records 

67 associated with a single `DimensionElement`. 

68 

69 Concrete `DimensionRecordStorage` instances should generally be constructed 

70 via a call to `setupDimensionStorage`, which selects the appropriate 

71 subclass for each element according to its configuration. 

72 

73 All `DimensionRecordStorage` methods are pure abstract, even though in some 

74 cases a reasonable default implementation might be possible, in order to 

75 better guarantee all methods are correctly overridden. All of these 

76 potentially-defaultable implementations are extremely trivial, so asking 

77 subclasses to provide them is not a significant burden. 

78 """ 

79 

80 @property 

81 @abstractmethod 

82 def element(self) -> DimensionElement: 

83 """The element whose records this instance holds (`DimensionElement`). 

84 """ 

85 raise NotImplementedError() 

86 

87 @abstractmethod 

88 def clearCaches(self) -> None: 

89 """Clear any in-memory caches held by the storage instance. 

90 

91 This is called by `Registry` when transactions are rolled back, to 

92 avoid in-memory caches from ever containing records that are not 

93 present in persistent storage. 

94 """ 

95 raise NotImplementedError() 

96 

97 @abstractmethod 

98 def join( 

99 self, 

100 builder: QueryBuilder, *, 

101 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

102 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None, 

103 ) -> sqlalchemy.sql.FromClause: 

104 """Add the dimension element's logical table to a query under 

105 construction. 

106 

107 This is a visitor pattern interface that is expected to be called only 

108 by `QueryBuilder.joinDimensionElement`. 

109 

110 Parameters 

111 ---------- 

112 builder : `QueryBuilder` 

113 Builder for the query that should contain this element. 

114 regions : `NamedKeyDict`, optional 

115 A mapping from `DimensionElement` to a SQLAlchemy column containing 

116 the region for that element, which should be updated to include a 

117 region column for this element if one exists. If `None`, 

118 ``self.element`` is not being included in the query via a spatial 

119 join. 

120 timespan : `NamedKeyDict`, optional 

121 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

122 columns containing the timespan for that element, which should be 

123 updated to include timespan columns for this element if they exist. 

124 If `None`, ``self.element`` is not being included in the query via 

125 a temporal join. 

126 

127 Returns 

128 ------- 

129 fromClause : `sqlalchemy.sql.FromClause` 

130 Table or clause for the element which is joined. 

131 

132 Notes 

133 ----- 

134 Elements are only included in queries via spatial and/or temporal joins 

135 when necessary to connect them to other elements in the query, so 

136 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

137 because an element has a region or timespan. 

138 """ 

139 raise NotImplementedError() 

140 

141 @abstractmethod 

142 def insert(self, *records: DimensionRecord) -> None: 

143 """Insert one or more records into storage. 

144 

145 Parameters 

146 ---------- 

147 records 

148 One or more instances of the `DimensionRecord` subclass for the 

149 element this storage is associated with. 

150 

151 Raises 

152 ------ 

153 TypeError 

154 Raised if the element does not support record insertion. 

155 sqlalchemy.exc.IntegrityError 

156 Raised if one or more records violate database integrity 

157 constraints. 

158 

159 Notes 

160 ----- 

161 As `insert` is expected to be called only by a `Registry`, we rely 

162 on `Registry` to provide transactionality, both by using a SQLALchemy 

163 connection shared with the `Registry` and by relying on it to call 

164 `clearCaches` when rolling back transactions. 

165 """ 

166 raise NotImplementedError() 

167 

168 @abstractmethod 

169 def sync(self, record: DimensionRecord) -> bool: 

170 """Synchronize a record with the database, inserting it only if it does 

171 not exist and comparing values if it does. 

172 

173 Parameters 

174 ---------- 

175 record : `DimensionRecord`. 

176 An instance of the `DimensionRecord` subclass for the 

177 element this storage is associated with. 

178 

179 Returns 

180 ------- 

181 inserted : `bool` 

182 `True` if a new row was inserted, `False` otherwise. 

183 

184 Raises 

185 ------ 

186 DatabaseConflictError 

187 Raised if the record exists in the database (according to primary 

188 key lookup) but is inconsistent with the given one. 

189 TypeError 

190 Raised if the element does not support record synchronization. 

191 sqlalchemy.exc.IntegrityError 

192 Raised if one or more records violate database integrity 

193 constraints. 

194 """ 

195 raise NotImplementedError() 

196 

197 @abstractmethod 

198 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

199 """Retrieve records from storage. 

200 

201 Parameters 

202 ---------- 

203 dataIds : `DataCoordinateIterable` 

204 Data IDs that identify the records to be retrieved. 

205 

206 Returns 

207 ------- 

208 records : `Iterable` [ `DimensionRecord` ] 

209 Record retrieved from storage. Not all data IDs may have 

210 corresponding records (if there are no records that match a data 

211 ID), and even if they are, the order of inputs is not preserved. 

212 """ 

213 raise NotImplementedError() 

214 

215 @abstractmethod 

216 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

217 """Return tables used for schema digest. 

218 

219 Returns 

220 ------- 

221 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

222 Possibly empty set of tables for schema digest calculations. 

223 """ 

224 raise NotImplementedError() 

225 

226 

227class GovernorDimensionRecordStorage(DimensionRecordStorage): 

228 """Intermediate interface for `DimensionRecordStorage` objects that provide 

229 storage for `GovernorDimension` instances. 

230 """ 

231 

232 @classmethod 

233 @abstractmethod 

234 def initialize(cls, db: Database, dimension: GovernorDimension, *, 

235 context: Optional[StaticTablesContext] = None, 

236 config: Mapping[str, Any], 

237 ) -> GovernorDimensionRecordStorage: 

238 """Construct an instance of this class using a standardized interface. 

239 

240 Parameters 

241 ---------- 

242 db : `Database` 

243 Interface to the underlying database engine and namespace. 

244 dimension : `GovernorDimension` 

245 Dimension the new instance will manage records for. 

246 context : `StaticTablesContext`, optional 

247 If provided, an object to use to create any new tables. If not 

248 provided, ``db.ensureTableExists`` should be used instead. 

249 config : `Mapping` 

250 Extra configuration options specific to the implementation. 

251 

252 Returns 

253 ------- 

254 storage : `GovernorDimensionRecordStorage` 

255 A new `GovernorDimensionRecordStorage` subclass instance. 

256 """ 

257 raise NotImplementedError() 

258 

259 @property 

260 @abstractmethod 

261 def element(self) -> GovernorDimension: 

262 # Docstring inherited from DimensionRecordStorage. 

263 raise NotImplementedError() 

264 

265 @abstractmethod 

266 def refresh(self) -> None: 

267 """Ensure all other operations on this manager are aware of any 

268 changes made by other clients since it was initialized or last 

269 refreshed. 

270 """ 

271 raise NotImplementedError() 

272 

273 @property 

274 @abstractmethod 

275 def values(self) -> AbstractSet[str]: 

276 """All primary key values for this dimension (`set` [ `str` ]). 

277 

278 This may rely on an in-memory cache and hence not reflect changes to 

279 the set of values made by other `Butler` / `Registry` clients. Call 

280 `refresh` to ensure up-to-date results. 

281 """ 

282 raise NotImplementedError() 

283 

284 @property 

285 @abstractmethod 

286 def table(self) -> sqlalchemy.schema.Table: 

287 """The SQLAlchemy table that backs this dimension 

288 (`sqlalchemy.schema.Table`). 

289 """ 

290 raise NotImplementedError 

291 

292 @abstractmethod 

293 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None: 

294 """Add a function or method to be called after new records for this 

295 dimension are inserted by `insert` or `sync`. 

296 

297 Parameters 

298 ---------- 

299 callback 

300 Callable that takes a single `DimensionRecord` argument. This will 

301 be called immediately after any successful insertion, in the same 

302 transaction. 

303 """ 

304 raise NotImplementedError() 

305 

306 

307class SkyPixDimensionRecordStorage(DimensionRecordStorage): 

308 """Intermediate interface for `DimensionRecordStorage` objects that provide 

309 storage for `SkyPixDimension` instances. 

310 """ 

311 

312 @property 

313 @abstractmethod 

314 def element(self) -> SkyPixDimension: 

315 # Docstring inherited from DimensionRecordStorage. 

316 raise NotImplementedError() 

317 

318 

319class DatabaseDimensionRecordStorage(DimensionRecordStorage): 

320 """Intermediate interface for `DimensionRecordStorage` objects that provide 

321 storage for `DatabaseDimensionElement` instances. 

322 """ 

323 

324 @classmethod 

325 @abstractmethod 

326 def initialize(cls, db: Database, element: DatabaseDimensionElement, *, 

327 context: Optional[StaticTablesContext] = None, 

328 config: Mapping[str, Any], 

329 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

330 ) -> DatabaseDimensionRecordStorage: 

331 """Construct an instance of this class using a standardized interface. 

332 

333 Parameters 

334 ---------- 

335 db : `Database` 

336 Interface to the underlying database engine and namespace. 

337 element : `DatabaseDimensionElement` 

338 Dimension element the new instance will manage records for. 

339 context : `StaticTablesContext`, optional 

340 If provided, an object to use to create any new tables. If not 

341 provided, ``db.ensureTableExists`` should be used instead. 

342 config : `Mapping` 

343 Extra configuration options specific to the implementation. 

344 governors : `NamedKeyMapping` 

345 Mapping containing all governor dimension storage implementations. 

346 

347 Returns 

348 ------- 

349 storage : `DatabaseDimensionRecordStorage` 

350 A new `DatabaseDimensionRecordStorage` subclass instance. 

351 """ 

352 raise NotImplementedError() 

353 

354 @property 

355 @abstractmethod 

356 def element(self) -> DatabaseDimensionElement: 

357 # Docstring inherited from DimensionRecordStorage. 

358 raise NotImplementedError() 

359 

360 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None: 

361 """Inform this record storage object of the object that will manage 

362 the overlaps between this element and another element. 

363 

364 This will only be called if ``self.element.spatial is not None``, 

365 and will be called immediately after construction (before any other 

366 methods). In the future, implementations will be required to call a 

367 method on any connected overlap storage objects any time new records 

368 for the element are inserted. 

369 

370 Parameters 

371 ---------- 

372 overlaps : `DatabaseDimensionRecordStorage` 

373 Object managing overlaps between this element and another 

374 database-backed element. 

375 """ 

376 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.") 

377 

378 

379class DatabaseDimensionOverlapStorage(ABC): 

380 """A base class for objects that manage overlaps between a pair of 

381 database-backed dimensions. 

382 """ 

383 

384 @classmethod 

385 @abstractmethod 

386 def initialize( 

387 cls, 

388 db: Database, 

389 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage], 

390 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage], 

391 context: Optional[StaticTablesContext] = None, 

392 ) -> DatabaseDimensionOverlapStorage: 

393 """Construct an instance of this class using a standardized interface. 

394 

395 Parameters 

396 ---------- 

397 db : `Database` 

398 Interface to the underlying database engine and namespace. 

399 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ] 

400 Storage objects for the elements this object will related. 

401 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ] 

402 Storage objects for the governor dimensions of the elements this 

403 object will related. 

404 context : `StaticTablesContext`, optional 

405 If provided, an object to use to create any new tables. If not 

406 provided, ``db.ensureTableExists`` should be used instead. 

407 

408 Returns 

409 ------- 

410 storage : `DatabaseDimensionOverlapStorage` 

411 A new `DatabaseDimensionOverlapStorage` subclass instance. 

412 """ 

413 raise NotImplementedError() 

414 

415 @property 

416 @abstractmethod 

417 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]: 

418 """The pair of elements whose overlaps this object manages. 

419 

420 The order of elements is the same as their ordering within the 

421 `DimensionUniverse`. 

422 """ 

423 raise NotImplementedError() 

424 

425 @abstractmethod 

426 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

427 """Return tables used for schema digest. 

428 

429 Returns 

430 ------- 

431 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

432 Possibly empty set of tables for schema digest calculations. 

433 """ 

434 raise NotImplementedError() 

435 

436 

437class DimensionRecordStorageManager(VersionedExtension): 

438 """An interface for managing the dimension records in a `Registry`. 

439 

440 `DimensionRecordStorageManager` primarily serves as a container and factory 

441 for `DimensionRecordStorage` instances, which each provide access to the 

442 records for a different `DimensionElement`. 

443 

444 Parameters 

445 ---------- 

446 universe : `DimensionUniverse` 

447 Universe of all dimensions and dimension elements known to the 

448 `Registry`. 

449 

450 Notes 

451 ----- 

452 In a multi-layer `Registry`, many dimension elements will only have 

453 records in one layer (often the base layer). The union of the records 

454 across all layers forms the logical table for the full `Registry`. 

455 """ 

456 def __init__(self, *, universe: DimensionUniverse): 

457 self.universe = universe 

458 

459 @classmethod 

460 @abstractmethod 

461 def initialize(cls, db: Database, context: StaticTablesContext, *, 

462 universe: DimensionUniverse) -> DimensionRecordStorageManager: 

463 """Construct an instance of the manager. 

464 

465 Parameters 

466 ---------- 

467 db : `Database` 

468 Interface to the underlying database engine and namespace. 

469 context : `StaticTablesContext` 

470 Context object obtained from `Database.declareStaticTables`; used 

471 to declare any tables that should always be present in a layer 

472 implemented with this manager. 

473 universe : `DimensionUniverse` 

474 Universe graph containing dimensions known to this `Registry`. 

475 

476 Returns 

477 ------- 

478 manager : `DimensionRecordStorageManager` 

479 An instance of a concrete `DimensionRecordStorageManager` subclass. 

480 """ 

481 raise NotImplementedError() 

482 

483 @abstractmethod 

484 def refresh(self) -> None: 

485 """Ensure all other operations on this manager are aware of any 

486 changes made by other clients since it was initialized or last 

487 refreshed. 

488 """ 

489 raise NotImplementedError() 

490 

491 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

492 """Interface to `get` that raises `LookupError` instead of returning 

493 `None` on failure. 

494 """ 

495 r = self.get(element) 

496 if r is None: 

497 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

498 return r 

499 

500 @abstractmethod 

501 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

502 """Return an object that provides access to the records associated with 

503 the given element, if one exists in this layer. 

504 

505 Parameters 

506 ---------- 

507 element : `DimensionElement` 

508 Element for which records should be returned. 

509 

510 Returns 

511 ------- 

512 records : `DimensionRecordStorage` or `None` 

513 The object representing the records for the given element in this 

514 layer, or `None` if there are no records for that element in this 

515 layer. 

516 

517 Notes 

518 ----- 

519 Dimension elements registered by another client of the same layer since 

520 the last call to `initialize` or `refresh` may not be found. 

521 """ 

522 raise NotImplementedError() 

523 

524 @abstractmethod 

525 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

526 """Ensure that this layer can hold records for the given element, 

527 creating new tables as necessary. 

528 

529 Parameters 

530 ---------- 

531 element : `DimensionElement` 

532 Element for which a table should created (as necessary) and 

533 an associated `DimensionRecordStorage` returned. 

534 

535 Returns 

536 ------- 

537 records : `DimensionRecordStorage` 

538 The object representing the records for the given element in this 

539 layer. 

540 

541 Raises 

542 ------ 

543 TransactionInterruption 

544 Raised if this operation is invoked within a `Database.transaction` 

545 context. 

546 """ 

547 raise NotImplementedError() 

548 

549 @abstractmethod 

550 def saveDimensionGraph(self, graph: DimensionGraph) -> int: 

551 """Save a `DimensionGraph` definition to the database, allowing it to 

552 be retrieved later via the returned key. 

553 

554 Parameters 

555 ---------- 

556 graph : `DimensionGraph` 

557 Set of dimensions to save. 

558 

559 Returns 

560 ------- 

561 key : `int` 

562 Integer used as the unique key for this `DimensionGraph` in the 

563 database. 

564 

565 Raises 

566 ------ 

567 TransactionInterruption 

568 Raised if this operation is invoked within a `Database.transaction` 

569 context. 

570 """ 

571 raise NotImplementedError() 

572 

573 @abstractmethod 

574 def loadDimensionGraph(self, key: int) -> DimensionGraph: 

575 """Retrieve a `DimensionGraph` that was previously saved in the 

576 database. 

577 

578 Parameters 

579 ---------- 

580 key : `int` 

581 Integer used as the unique key for this `DimensionGraph` in the 

582 database. 

583 

584 Returns 

585 ------- 

586 graph : `DimensionGraph` 

587 Retrieved graph. 

588 

589 Raises 

590 ------ 

591 KeyError 

592 Raised if the given key cannot be found in the database. 

593 """ 

594 raise NotImplementedError() 

595 

596 @abstractmethod 

597 def clearCaches(self) -> None: 

598 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

599 instances. 

600 

601 This is called by `Registry` when transactions are rolled back, to 

602 avoid in-memory caches from ever containing records that are not 

603 present in persistent storage. 

604 """ 

605 raise NotImplementedError() 

606 

607 universe: DimensionUniverse 

608 """Universe of all dimensions and dimension elements known to the 

609 `Registry` (`DimensionUniverse`). 

610 """