Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "DatabaseDimensionOverlapStorage", 

25 "DatabaseDimensionRecordStorage", 

26 "DimensionRecordStorage", 

27 "DimensionRecordStorageManager", 

28 "GovernorDimensionRecordStorage", 

29 "SkyPixDimensionRecordStorage", 

30) 

31 

32from abc import ABC, abstractmethod 

33from typing import ( 

34 AbstractSet, 

35 Callable, 

36 Iterable, 

37 Optional, 

38 Tuple, 

39 TYPE_CHECKING, 

40 Union, 

41) 

42 

43import sqlalchemy 

44 

45from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension 

46from ._versioning import VersionedExtension 

47 

48if TYPE_CHECKING: 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true

49 from ...core import ( 

50 Config, 

51 DataCoordinateIterable, 

52 DimensionElement, 

53 DimensionRecord, 

54 DimensionUniverse, 

55 NamedKeyDict, 

56 NamedKeyMapping, 

57 TimespanDatabaseRepresentation, 

58 ) 

59 from ..queries import QueryBuilder 

60 from ._database import Database, StaticTablesContext 

61 

62 

63OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]] 

64 

65 

66class DimensionRecordStorage(ABC): 

67 """An abstract base class that represents a way of storing the records 

68 associated with a single `DimensionElement`. 

69 

70 Concrete `DimensionRecordStorage` instances should generally be constructed 

71 via a call to `setupDimensionStorage`, which selects the appropriate 

72 subclass for each element according to its configuration. 

73 

74 All `DimensionRecordStorage` methods are pure abstract, even though in some 

75 cases a reasonable default implementation might be possible, in order to 

76 better guarantee all methods are correctly overridden. All of these 

77 potentially-defaultable implementations are extremely trivial, so asking 

78 subclasses to provide them is not a significant burden. 

79 """ 

80 

81 @property 

82 @abstractmethod 

83 def element(self) -> DimensionElement: 

84 """The element whose records this instance holds (`DimensionElement`). 

85 """ 

86 raise NotImplementedError() 

87 

88 @abstractmethod 

89 def clearCaches(self) -> None: 

90 """Clear any in-memory caches held by the storage instance. 

91 

92 This is called by `Registry` when transactions are rolled back, to 

93 avoid in-memory caches from ever containing records that are not 

94 present in persistent storage. 

95 """ 

96 raise NotImplementedError() 

97 

98 @abstractmethod 

99 def join( 

100 self, 

101 builder: QueryBuilder, *, 

102 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

103 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None, 

104 ) -> sqlalchemy.sql.FromClause: 

105 """Add the dimension element's logical table to a query under 

106 construction. 

107 

108 This is a visitor pattern interface that is expected to be called only 

109 by `QueryBuilder.joinDimensionElement`. 

110 

111 Parameters 

112 ---------- 

113 builder : `QueryBuilder` 

114 Builder for the query that should contain this element. 

115 regions : `NamedKeyDict`, optional 

116 A mapping from `DimensionElement` to a SQLAlchemy column containing 

117 the region for that element, which should be updated to include a 

118 region column for this element if one exists. If `None`, 

119 ``self.element`` is not being included in the query via a spatial 

120 join. 

121 timespan : `NamedKeyDict`, optional 

122 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

123 columns containing the timespan for that element, which should be 

124 updated to include timespan columns for this element if they exist. 

125 If `None`, ``self.element`` is not being included in the query via 

126 a temporal join. 

127 

128 Returns 

129 ------- 

130 fromClause : `sqlalchemy.sql.FromClause` 

131 Table or clause for the element which is joined. 

132 

133 Notes 

134 ----- 

135 Elements are only included in queries via spatial and/or temporal joins 

136 when necessary to connect them to other elements in the query, so 

137 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

138 because an element has a region or timespan. 

139 """ 

140 raise NotImplementedError() 

141 

142 @abstractmethod 

143 def insert(self, *records: DimensionRecord) -> None: 

144 """Insert one or more records into storage. 

145 

146 Parameters 

147 ---------- 

148 records 

149 One or more instances of the `DimensionRecord` subclass for the 

150 element this storage is associated with. 

151 

152 Raises 

153 ------ 

154 TypeError 

155 Raised if the element does not support record insertion. 

156 sqlalchemy.exc.IntegrityError 

157 Raised if one or more records violate database integrity 

158 constraints. 

159 

160 Notes 

161 ----- 

162 As `insert` is expected to be called only by a `Registry`, we rely 

163 on `Registry` to provide transactionality, both by using a SQLALchemy 

164 connection shared with the `Registry` and by relying on it to call 

165 `clearCaches` when rolling back transactions. 

166 """ 

167 raise NotImplementedError() 

168 

169 @abstractmethod 

170 def sync(self, record: DimensionRecord) -> bool: 

171 """Synchronize a record with the database, inserting it only if it does 

172 not exist and comparing values if it does. 

173 

174 Parameters 

175 ---------- 

176 record : `DimensionRecord`. 

177 An instance of the `DimensionRecord` subclass for the 

178 element this storage is associated with. 

179 

180 Returns 

181 ------- 

182 inserted : `bool` 

183 `True` if a new row was inserted, `False` otherwise. 

184 

185 Raises 

186 ------ 

187 DatabaseConflictError 

188 Raised if the record exists in the database (according to primary 

189 key lookup) but is inconsistent with the given one. 

190 TypeError 

191 Raised if the element does not support record synchronization. 

192 sqlalchemy.exc.IntegrityError 

193 Raised if one or more records violate database integrity 

194 constraints. 

195 """ 

196 raise NotImplementedError() 

197 

198 @abstractmethod 

199 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

200 """Retrieve records from storage. 

201 

202 Parameters 

203 ---------- 

204 dataIds : `DataCoordinateIterable` 

205 Data IDs that identify the records to be retrieved. 

206 

207 Returns 

208 ------- 

209 records : `Iterable` [ `DimensionRecord` ] 

210 Record retrieved from storage. Not all data IDs may have 

211 corresponding records (if there are no records that match a data 

212 ID), and even if they are, the order of inputs is not preserved. 

213 """ 

214 raise NotImplementedError() 

215 

216 @abstractmethod 

217 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

218 """Return tables used for schema digest. 

219 

220 Returns 

221 ------- 

222 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

223 Possibly empty set of tables for schema digest calculations. 

224 """ 

225 raise NotImplementedError() 

226 

227 

228class GovernorDimensionRecordStorage(DimensionRecordStorage): 

229 """Intermediate interface for `DimensionRecordStorage` objects that provide 

230 storage for `GovernorDimension` instances. 

231 """ 

232 

233 @classmethod 

234 @abstractmethod 

235 def initialize(cls, db: Database, dimension: GovernorDimension, *, 

236 context: Optional[StaticTablesContext] = None, 

237 config: Config, 

238 ) -> GovernorDimensionRecordStorage: 

239 """Construct an instance of this class using a standardized interface. 

240 

241 Parameters 

242 ---------- 

243 db : `Database` 

244 Interface to the underlying database engine and namespace. 

245 dimension : `GovernorDimension` 

246 Dimension the new instance will manage records for. 

247 context : `StaticTablesContext`, optional 

248 If provided, an object to use to create any new tables. If not 

249 provided, ``db.ensureTableExists`` should be used instead. 

250 config : `Config` 

251 Extra configuration options specific to the implementation. 

252 

253 Returns 

254 ------- 

255 storage : `GovernorDimensionRecordStorage` 

256 A new `GovernorDimensionRecordStorage` subclass instance. 

257 """ 

258 raise NotImplementedError() 

259 

260 @property 

261 @abstractmethod 

262 def element(self) -> GovernorDimension: 

263 # Docstring inherited from DimensionRecordStorage. 

264 raise NotImplementedError() 

265 

266 @abstractmethod 

267 def refresh(self) -> None: 

268 """Ensure all other operations on this manager are aware of any 

269 changes made by other clients since it was initialized or last 

270 refreshed. 

271 """ 

272 raise NotImplementedError() 

273 

274 @property 

275 @abstractmethod 

276 def values(self) -> AbstractSet[str]: 

277 """All primary key values for this dimension (`set` [ `str` ]). 

278 

279 This may rely on an in-memory cache and hence not reflect changes to 

280 the set of values made by other `Butler` / `Registry` clients. Call 

281 `refresh` to ensure up-to-date results. 

282 """ 

283 raise NotImplementedError() 

284 

285 @property 

286 @abstractmethod 

287 def table(self) -> sqlalchemy.schema.Table: 

288 """The SQLAlchemy table that backs this dimension 

289 (`sqlalchemy.schema.Table`). 

290 """ 

291 raise NotImplementedError 

292 

293 @abstractmethod 

294 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None: 

295 """Add a function or method to be called after new records for this 

296 dimension are inserted by `insert` or `sync`. 

297 

298 Parameters 

299 ---------- 

300 callback 

301 Callable that takes a single `DimensionRecord` argument. This will 

302 be called immediately after any successful insertion, in the same 

303 transaction. 

304 """ 

305 raise NotImplementedError() 

306 

307 

308class SkyPixDimensionRecordStorage(DimensionRecordStorage): 

309 """Intermediate interface for `DimensionRecordStorage` objects that provide 

310 storage for `SkyPixDimension` instances. 

311 """ 

312 

313 @property 

314 @abstractmethod 

315 def element(self) -> SkyPixDimension: 

316 # Docstring inherited from DimensionRecordStorage. 

317 raise NotImplementedError() 

318 

319 

320class DatabaseDimensionRecordStorage(DimensionRecordStorage): 

321 """Intermediate interface for `DimensionRecordStorage` objects that provide 

322 storage for `DatabaseDimensionElement` instances. 

323 """ 

324 

325 @classmethod 

326 @abstractmethod 

327 def initialize(cls, db: Database, element: DatabaseDimensionElement, *, 

328 context: Optional[StaticTablesContext] = None, 

329 config: Config, 

330 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

331 ) -> DatabaseDimensionRecordStorage: 

332 """Construct an instance of this class using a standardized interface. 

333 

334 Parameters 

335 ---------- 

336 db : `Database` 

337 Interface to the underlying database engine and namespace. 

338 element : `DatabaseDimensionElement` 

339 Dimension element the new instance will manage records for. 

340 context : `StaticTablesContext`, optional 

341 If provided, an object to use to create any new tables. If not 

342 provided, ``db.ensureTableExists`` should be used instead. 

343 config : `Config` 

344 Extra configuration options specific to the implementation. 

345 governors : `NamedKeyMapping` 

346 Mapping containing all governor dimension storage implementations. 

347 

348 Returns 

349 ------- 

350 storage : `DatabaseDimensionRecordStorage` 

351 A new `DatabaseDimensionRecordStorage` subclass instance. 

352 """ 

353 raise NotImplementedError() 

354 

355 @property 

356 @abstractmethod 

357 def element(self) -> DatabaseDimensionElement: 

358 # Docstring inherited from DimensionRecordStorage. 

359 raise NotImplementedError() 

360 

361 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None: 

362 """Inform this record storage object of the object that will manage 

363 the overlaps between this element and another element. 

364 

365 This will only be called if ``self.element.spatial is not None``, 

366 and will be called immediately after construction (before any other 

367 methods). In the future, implementations will be required to call a 

368 method on any connected overlap storage objects any time new records 

369 for the element are inserted. 

370 

371 Parameters 

372 ---------- 

373 overlaps : `DatabaseDimensionRecordStorage` 

374 Object managing overlaps between this element and another 

375 database-backed element. 

376 """ 

377 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.") 

378 

379 

380class DatabaseDimensionOverlapStorage(ABC): 

381 """A base class for objects that manage overlaps between a pair of 

382 database-backed dimensions. 

383 """ 

384 

385 @classmethod 

386 @abstractmethod 

387 def initialize( 

388 cls, 

389 db: Database, 

390 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage], 

391 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage], 

392 context: Optional[StaticTablesContext] = None, 

393 ) -> DatabaseDimensionOverlapStorage: 

394 """Construct an instance of this class using a standardized interface. 

395 

396 Parameters 

397 ---------- 

398 db : `Database` 

399 Interface to the underlying database engine and namespace. 

400 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ] 

401 Storage objects for the elements this object will related. 

402 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ] 

403 Storage objects for the governor dimensions of the elements this 

404 object will related. 

405 context : `StaticTablesContext`, optional 

406 If provided, an object to use to create any new tables. If not 

407 provided, ``db.ensureTableExists`` should be used instead. 

408 

409 Returns 

410 ------- 

411 storage : `DatabaseDimensionOverlapStorage` 

412 A new `DatabaseDimensionOverlapStorage` subclass instance. 

413 """ 

414 raise NotImplementedError() 

415 

416 @property 

417 @abstractmethod 

418 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]: 

419 """The pair of elements whose overlaps this object manages. 

420 

421 The order of elements is the same as their ordering within the 

422 `DimensionUniverse`. 

423 """ 

424 raise NotImplementedError() 

425 

426 @abstractmethod 

427 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

428 """Return tables used for schema digest. 

429 

430 Returns 

431 ------- 

432 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

433 Possibly empty set of tables for schema digest calculations. 

434 """ 

435 raise NotImplementedError() 

436 

437 

438class DimensionRecordStorageManager(VersionedExtension): 

439 """An interface for managing the dimension records in a `Registry`. 

440 

441 `DimensionRecordStorageManager` primarily serves as a container and factory 

442 for `DimensionRecordStorage` instances, which each provide access to the 

443 records for a different `DimensionElement`. 

444 

445 Parameters 

446 ---------- 

447 universe : `DimensionUniverse` 

448 Universe of all dimensions and dimension elements known to the 

449 `Registry`. 

450 

451 Notes 

452 ----- 

453 In a multi-layer `Registry`, many dimension elements will only have 

454 records in one layer (often the base layer). The union of the records 

455 across all layers forms the logical table for the full `Registry`. 

456 """ 

457 def __init__(self, *, universe: DimensionUniverse): 

458 self.universe = universe 

459 

460 @classmethod 

461 @abstractmethod 

462 def initialize(cls, db: Database, context: StaticTablesContext, *, 

463 universe: DimensionUniverse) -> DimensionRecordStorageManager: 

464 """Construct an instance of the manager. 

465 

466 Parameters 

467 ---------- 

468 db : `Database` 

469 Interface to the underlying database engine and namespace. 

470 context : `StaticTablesContext` 

471 Context object obtained from `Database.declareStaticTables`; used 

472 to declare any tables that should always be present in a layer 

473 implemented with this manager. 

474 universe : `DimensionUniverse` 

475 Universe graph containing dimensions known to this `Registry`. 

476 

477 Returns 

478 ------- 

479 manager : `DimensionRecordStorageManager` 

480 An instance of a concrete `DimensionRecordStorageManager` subclass. 

481 """ 

482 raise NotImplementedError() 

483 

484 @abstractmethod 

485 def refresh(self) -> None: 

486 """Ensure all other operations on this manager are aware of any 

487 changes made by other clients since it was initialized or last 

488 refreshed. 

489 """ 

490 raise NotImplementedError() 

491 

492 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

493 """Interface to `get` that raises `LookupError` instead of returning 

494 `None` on failure. 

495 """ 

496 r = self.get(element) 

497 if r is None: 

498 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

499 return r 

500 

501 @abstractmethod 

502 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

503 """Return an object that provides access to the records associated with 

504 the given element, if one exists in this layer. 

505 

506 Parameters 

507 ---------- 

508 element : `DimensionElement` 

509 Element for which records should be returned. 

510 

511 Returns 

512 ------- 

513 records : `DimensionRecordStorage` or `None` 

514 The object representing the records for the given element in this 

515 layer, or `None` if there are no records for that element in this 

516 layer. 

517 

518 Notes 

519 ----- 

520 Dimension elements registered by another client of the same layer since 

521 the last call to `initialize` or `refresh` may not be found. 

522 """ 

523 raise NotImplementedError() 

524 

525 @abstractmethod 

526 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

527 """Ensure that this layer can hold records for the given element, 

528 creating new tables as necessary. 

529 

530 Parameters 

531 ---------- 

532 element : `DimensionElement` 

533 Element for which a table should created (as necessary) and 

534 an associated `DimensionRecordStorage` returned. 

535 

536 Returns 

537 ------- 

538 records : `DimensionRecordStorage` 

539 The object representing the records for the given element in this 

540 layer. 

541 

542 Raises 

543 ------ 

544 TransactionInterruption 

545 Raised if this operation is invoked within a `Database.transaction` 

546 context. 

547 """ 

548 raise NotImplementedError() 

549 

550 @abstractmethod 

551 def saveDimensionGraph(self, graph: DimensionGraph) -> int: 

552 """Save a `DimensionGraph` definition to the database, allowing it to 

553 be retrieved later via the returned key. 

554 

555 Parameters 

556 ---------- 

557 graph : `DimensionGraph` 

558 Set of dimensions to save. 

559 

560 Returns 

561 ------- 

562 key : `int` 

563 Integer used as the unique key for this `DimensionGraph` in the 

564 database. 

565 

566 Raises 

567 ------ 

568 TransactionInterruption 

569 Raised if this operation is invoked within a `Database.transaction` 

570 context. 

571 """ 

572 raise NotImplementedError() 

573 

574 @abstractmethod 

575 def loadDimensionGraph(self, key: int) -> DimensionGraph: 

576 """Retrieve a `DimensionGraph` that was previously saved in the 

577 database. 

578 

579 Parameters 

580 ---------- 

581 key : `int` 

582 Integer used as the unique key for this `DimensionGraph` in the 

583 database. 

584 

585 Returns 

586 ------- 

587 graph : `DimensionGraph` 

588 Retrieved graph. 

589 

590 Raises 

591 ------ 

592 KeyError 

593 Raised if the given key cannot be found in the database. 

594 """ 

595 raise NotImplementedError() 

596 

597 @abstractmethod 

598 def clearCaches(self) -> None: 

599 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

600 instances. 

601 

602 This is called by `Registry` when transactions are rolled back, to 

603 avoid in-memory caches from ever containing records that are not 

604 present in persistent storage. 

605 """ 

606 raise NotImplementedError() 

607 

608 universe: DimensionUniverse 

609 """Universe of all dimensions and dimension elements known to the 

610 `Registry` (`DimensionUniverse`). 

611 """