Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 69%

119 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-15 10:02 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "DatabaseDimensionOverlapStorage", 

25 "DatabaseDimensionRecordStorage", 

26 "DimensionRecordStorage", 

27 "DimensionRecordStorageManager", 

28 "GovernorDimensionRecordStorage", 

29 "SkyPixDimensionRecordStorage", 

30) 

31 

32from abc import ABC, abstractmethod 

33from collections.abc import Callable, Iterable, Mapping, Set 

34from typing import TYPE_CHECKING, Any, Tuple, Union 

35 

36import sqlalchemy 

37 

38from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension 

39from ._versioning import VersionedExtension 

40 

41if TYPE_CHECKING: 41 ↛ 42line 41 didn't jump to line 42, because the condition on line 41 was never true

42 from ...core import ( 

43 DataCoordinateIterable, 

44 DimensionElement, 

45 DimensionRecord, 

46 DimensionUniverse, 

47 NamedKeyDict, 

48 NamedKeyMapping, 

49 TimespanDatabaseRepresentation, 

50 ) 

51 from ..queries import QueryBuilder 

52 from ._database import Database, StaticTablesContext 

53 

54 

55OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]] 

56 

57 

58class DimensionRecordStorage(ABC): 

59 """An abstract base class that represents a way of storing the records 

60 associated with a single `DimensionElement`. 

61 

62 Concrete `DimensionRecordStorage` instances should generally be constructed 

63 via a call to `setupDimensionStorage`, which selects the appropriate 

64 subclass for each element according to its configuration. 

65 

66 All `DimensionRecordStorage` methods are pure abstract, even though in some 

67 cases a reasonable default implementation might be possible, in order to 

68 better guarantee all methods are correctly overridden. All of these 

69 potentially-defaultable implementations are extremely trivial, so asking 

70 subclasses to provide them is not a significant burden. 

71 """ 

72 

73 @property 

74 @abstractmethod 

75 def element(self) -> DimensionElement: 

76 """The element whose records this instance managers 

77 (`DimensionElement`). 

78 """ 

79 raise NotImplementedError() 

80 

81 @abstractmethod 

82 def clearCaches(self) -> None: 

83 """Clear any in-memory caches held by the storage instance. 

84 

85 This is called by `Registry` when transactions are rolled back, to 

86 avoid in-memory caches from ever containing records that are not 

87 present in persistent storage. 

88 """ 

89 raise NotImplementedError() 

90 

91 @abstractmethod 

92 def join( 

93 self, 

94 builder: QueryBuilder, 

95 *, 

96 regions: NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement] | None = None, 

97 timespans: NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation] | None = None, 

98 ) -> sqlalchemy.sql.FromClause: 

99 """Add the dimension element's logical table to a query under 

100 construction. 

101 

102 This is a visitor pattern interface that is expected to be called only 

103 by `QueryBuilder.joinDimensionElement`. 

104 

105 Parameters 

106 ---------- 

107 builder : `QueryBuilder` 

108 Builder for the query that should contain this element. 

109 regions : `NamedKeyDict`, optional 

110 A mapping from `DimensionElement` to a SQLAlchemy column containing 

111 the region for that element, which should be updated to include a 

112 region column for this element if one exists. If `None`, 

113 ``self.element`` is not being included in the query via a spatial 

114 join. 

115 timespan : `NamedKeyDict`, optional 

116 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

117 columns containing the timespan for that element, which should be 

118 updated to include timespan columns for this element if they exist. 

119 If `None`, ``self.element`` is not being included in the query via 

120 a temporal join. 

121 

122 Returns 

123 ------- 

124 fromClause : `sqlalchemy.sql.FromClause` 

125 Table or clause for the element which is joined. 

126 

127 Notes 

128 ----- 

129 Elements are only included in queries via spatial and/or temporal joins 

130 when necessary to connect them to other elements in the query, so 

131 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

132 because an element has a region or timespan. 

133 """ 

134 raise NotImplementedError() 

135 

136 @abstractmethod 

137 def insert(self, *records: DimensionRecord, replace: bool = False, skip_existing: bool = False) -> None: 

138 """Insert one or more records into storage. 

139 

140 Parameters 

141 ---------- 

142 records 

143 One or more instances of the `DimensionRecord` subclass for the 

144 element this storage is associated with. 

145 replace: `bool`, optional 

146 If `True` (`False` is default), replace existing records in the 

147 database if there is a conflict. 

148 skip_existing : `bool`, optional 

149 If `True` (`False` is default), skip insertion if a record with 

150 the same primary key values already exists. 

151 

152 Raises 

153 ------ 

154 TypeError 

155 Raised if the element does not support record insertion. 

156 sqlalchemy.exc.IntegrityError 

157 Raised if one or more records violate database integrity 

158 constraints. 

159 

160 Notes 

161 ----- 

162 As `insert` is expected to be called only by a `Registry`, we rely 

163 on `Registry` to provide transactionality, both by using a SQLALchemy 

164 connection shared with the `Registry` and by relying on it to call 

165 `clearCaches` when rolling back transactions. 

166 """ 

167 raise NotImplementedError() 

168 

169 @abstractmethod 

170 def sync(self, record: DimensionRecord, update: bool = False) -> bool | dict[str, Any]: 

171 """Synchronize a record with the database, inserting it only if it does 

172 not exist and comparing values if it does. 

173 

174 Parameters 

175 ---------- 

176 record : `DimensionRecord`. 

177 An instance of the `DimensionRecord` subclass for the 

178 element this storage is associated with. 

179 update: `bool`, optional 

180 If `True` (`False` is default), update the existing record in the 

181 database if there is a conflict. 

182 

183 Returns 

184 ------- 

185 inserted_or_updated : `bool` or `dict` 

186 `True` if a new row was inserted, `False` if no changes were 

187 needed, or a `dict` mapping updated column names to their old 

188 values if an update was performed (only possible if 

189 ``update=True``). 

190 

191 Raises 

192 ------ 

193 DatabaseConflictError 

194 Raised if the record exists in the database (according to primary 

195 key lookup) but is inconsistent with the given one. 

196 TypeError 

197 Raised if the element does not support record synchronization. 

198 sqlalchemy.exc.IntegrityError 

199 Raised if one or more records violate database integrity 

200 constraints. 

201 """ 

202 raise NotImplementedError() 

203 

204 @abstractmethod 

205 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

206 """Retrieve records from storage. 

207 

208 Parameters 

209 ---------- 

210 dataIds : `DataCoordinateIterable` 

211 Data IDs that identify the records to be retrieved. 

212 

213 Returns 

214 ------- 

215 records : `Iterable` [ `DimensionRecord` ] 

216 Record retrieved from storage. Not all data IDs may have 

217 corresponding records (if there are no records that match a data 

218 ID), and even if they are, the order of inputs is not preserved. 

219 """ 

220 raise NotImplementedError() 

221 

222 @abstractmethod 

223 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

224 """Return tables used for schema digest. 

225 

226 Returns 

227 ------- 

228 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

229 Possibly empty set of tables for schema digest calculations. 

230 """ 

231 raise NotImplementedError() 

232 

233 

234class GovernorDimensionRecordStorage(DimensionRecordStorage): 

235 """Intermediate interface for `DimensionRecordStorage` objects that provide 

236 storage for `GovernorDimension` instances. 

237 """ 

238 

239 @classmethod 

240 @abstractmethod 

241 def initialize( 

242 cls, 

243 db: Database, 

244 dimension: GovernorDimension, 

245 *, 

246 context: StaticTablesContext | None = None, 

247 config: Mapping[str, Any], 

248 ) -> GovernorDimensionRecordStorage: 

249 """Construct an instance of this class using a standardized interface. 

250 

251 Parameters 

252 ---------- 

253 db : `Database` 

254 Interface to the underlying database engine and namespace. 

255 dimension : `GovernorDimension` 

256 Dimension the new instance will manage records for. 

257 context : `StaticTablesContext`, optional 

258 If provided, an object to use to create any new tables. If not 

259 provided, ``db.ensureTableExists`` should be used instead. 

260 config : `Mapping` 

261 Extra configuration options specific to the implementation. 

262 

263 Returns 

264 ------- 

265 storage : `GovernorDimensionRecordStorage` 

266 A new `GovernorDimensionRecordStorage` subclass instance. 

267 """ 

268 raise NotImplementedError() 

269 

270 @property 

271 @abstractmethod 

272 def element(self) -> GovernorDimension: 

273 # Docstring inherited from DimensionRecordStorage. 

274 raise NotImplementedError() 

275 

276 @abstractmethod 

277 def refresh(self) -> None: 

278 """Ensure all other operations on this manager are aware of any 

279 changes made by other clients since it was initialized or last 

280 refreshed. 

281 """ 

282 raise NotImplementedError() 

283 

284 @property 

285 @abstractmethod 

286 def values(self) -> Set[str]: 

287 """All primary key values for this dimension (`set` [ `str` ]). 

288 

289 This may rely on an in-memory cache and hence not reflect changes to 

290 the set of values made by other `Butler` / `Registry` clients. Call 

291 `refresh` to ensure up-to-date results. 

292 """ 

293 raise NotImplementedError() 

294 

295 @property 

296 @abstractmethod 

297 def table(self) -> sqlalchemy.schema.Table: 

298 """The SQLAlchemy table that backs this dimension 

299 (`sqlalchemy.schema.Table`). 

300 """ 

301 raise NotImplementedError 

302 

303 @abstractmethod 

304 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None: 

305 """Add a function or method to be called after new records for this 

306 dimension are inserted by `insert` or `sync`. 

307 

308 Parameters 

309 ---------- 

310 callback 

311 Callable that takes a single `DimensionRecord` argument. This will 

312 be called immediately after any successful insertion, in the same 

313 transaction. 

314 """ 

315 raise NotImplementedError() 

316 

317 

318class SkyPixDimensionRecordStorage(DimensionRecordStorage): 

319 """Intermediate interface for `DimensionRecordStorage` objects that provide 

320 storage for `SkyPixDimension` instances. 

321 """ 

322 

323 @property 

324 @abstractmethod 

325 def element(self) -> SkyPixDimension: 

326 # Docstring inherited from DimensionRecordStorage. 

327 raise NotImplementedError() 

328 

329 

330class DatabaseDimensionRecordStorage(DimensionRecordStorage): 

331 """Intermediate interface for `DimensionRecordStorage` objects that provide 

332 storage for `DatabaseDimensionElement` instances. 

333 """ 

334 

335 @classmethod 

336 @abstractmethod 

337 def initialize( 

338 cls, 

339 db: Database, 

340 element: DatabaseDimensionElement, 

341 *, 

342 context: StaticTablesContext | None = None, 

343 config: Mapping[str, Any], 

344 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

345 ) -> DatabaseDimensionRecordStorage: 

346 """Construct an instance of this class using a standardized interface. 

347 

348 Parameters 

349 ---------- 

350 db : `Database` 

351 Interface to the underlying database engine and namespace. 

352 element : `DatabaseDimensionElement` 

353 Dimension element the new instance will manage records for. 

354 context : `StaticTablesContext`, optional 

355 If provided, an object to use to create any new tables. If not 

356 provided, ``db.ensureTableExists`` should be used instead. 

357 config : `Mapping` 

358 Extra configuration options specific to the implementation. 

359 governors : `NamedKeyMapping` 

360 Mapping containing all governor dimension storage implementations. 

361 

362 Returns 

363 ------- 

364 storage : `DatabaseDimensionRecordStorage` 

365 A new `DatabaseDimensionRecordStorage` subclass instance. 

366 """ 

367 raise NotImplementedError() 

368 

369 @property 

370 @abstractmethod 

371 def element(self) -> DatabaseDimensionElement: 

372 # Docstring inherited from DimensionRecordStorage. 

373 raise NotImplementedError() 

374 

375 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None: 

376 """Inform this record storage object of the object that will manage 

377 the overlaps between this element and another element. 

378 

379 This will only be called if ``self.element.spatial is not None``, 

380 and will be called immediately after construction (before any other 

381 methods). In the future, implementations will be required to call a 

382 method on any connected overlap storage objects any time new records 

383 for the element are inserted. 

384 

385 Parameters 

386 ---------- 

387 overlaps : `DatabaseDimensionRecordStorage` 

388 Object managing overlaps between this element and another 

389 database-backed element. 

390 """ 

391 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.") 

392 

393 

394class DatabaseDimensionOverlapStorage(ABC): 

395 """A base class for objects that manage overlaps between a pair of 

396 database-backed dimensions. 

397 """ 

398 

399 @classmethod 

400 @abstractmethod 

401 def initialize( 

402 cls, 

403 db: Database, 

404 elementStorage: tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage], 

405 governorStorage: tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage], 

406 context: StaticTablesContext | None = None, 

407 ) -> DatabaseDimensionOverlapStorage: 

408 """Construct an instance of this class using a standardized interface. 

409 

410 Parameters 

411 ---------- 

412 db : `Database` 

413 Interface to the underlying database engine and namespace. 

414 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ] 

415 Storage objects for the elements this object will related. 

416 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ] 

417 Storage objects for the governor dimensions of the elements this 

418 object will related. 

419 context : `StaticTablesContext`, optional 

420 If provided, an object to use to create any new tables. If not 

421 provided, ``db.ensureTableExists`` should be used instead. 

422 

423 Returns 

424 ------- 

425 storage : `DatabaseDimensionOverlapStorage` 

426 A new `DatabaseDimensionOverlapStorage` subclass instance. 

427 """ 

428 raise NotImplementedError() 

429 

430 @property 

431 @abstractmethod 

432 def elements(self) -> tuple[DatabaseDimensionElement, DatabaseDimensionElement]: 

433 """The pair of elements whose overlaps this object manages. 

434 

435 The order of elements is the same as their ordering within the 

436 `DimensionUniverse`. 

437 """ 

438 raise NotImplementedError() 

439 

440 @abstractmethod 

441 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

442 """Return tables used for schema digest. 

443 

444 Returns 

445 ------- 

446 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

447 Possibly empty set of tables for schema digest calculations. 

448 """ 

449 raise NotImplementedError() 

450 

451 

452class DimensionRecordStorageManager(VersionedExtension): 

453 """An interface for managing the dimension records in a `Registry`. 

454 

455 `DimensionRecordStorageManager` primarily serves as a container and factory 

456 for `DimensionRecordStorage` instances, which each provide access to the 

457 records for a different `DimensionElement`. 

458 

459 Parameters 

460 ---------- 

461 universe : `DimensionUniverse` 

462 Universe of all dimensions and dimension elements known to the 

463 `Registry`. 

464 

465 Notes 

466 ----- 

467 In a multi-layer `Registry`, many dimension elements will only have 

468 records in one layer (often the base layer). The union of the records 

469 across all layers forms the logical table for the full `Registry`. 

470 """ 

471 

472 def __init__(self, *, universe: DimensionUniverse): 

473 self.universe = universe 

474 

475 @classmethod 

476 @abstractmethod 

477 def initialize( 

478 cls, db: Database, context: StaticTablesContext, *, universe: DimensionUniverse 

479 ) -> DimensionRecordStorageManager: 

480 """Construct an instance of the manager. 

481 

482 Parameters 

483 ---------- 

484 db : `Database` 

485 Interface to the underlying database engine and namespace. 

486 context : `StaticTablesContext` 

487 Context object obtained from `Database.declareStaticTables`; used 

488 to declare any tables that should always be present in a layer 

489 implemented with this manager. 

490 universe : `DimensionUniverse` 

491 Universe graph containing dimensions known to this `Registry`. 

492 

493 Returns 

494 ------- 

495 manager : `DimensionRecordStorageManager` 

496 An instance of a concrete `DimensionRecordStorageManager` subclass. 

497 """ 

498 raise NotImplementedError() 

499 

500 @abstractmethod 

501 def refresh(self) -> None: 

502 """Ensure all other operations on this manager are aware of any 

503 changes made by other clients since it was initialized or last 

504 refreshed. 

505 """ 

506 raise NotImplementedError() 

507 

508 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

509 """Interface to `get` that raises `LookupError` instead of returning 

510 `None` on failure. 

511 """ 

512 r = self.get(element) 

513 if r is None: 

514 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

515 return r 

516 

517 @abstractmethod 

518 def get(self, element: DimensionElement) -> DimensionRecordStorage | None: 

519 """Return an object that provides access to the records associated with 

520 the given element, if one exists in this layer. 

521 

522 Parameters 

523 ---------- 

524 element : `DimensionElement` 

525 Element for which records should be returned. 

526 

527 Returns 

528 ------- 

529 records : `DimensionRecordStorage` or `None` 

530 The object representing the records for the given element in this 

531 layer, or `None` if there are no records for that element in this 

532 layer. 

533 

534 Notes 

535 ----- 

536 Dimension elements registered by another client of the same layer since 

537 the last call to `initialize` or `refresh` may not be found. 

538 """ 

539 raise NotImplementedError() 

540 

541 @abstractmethod 

542 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

543 """Ensure that this layer can hold records for the given element, 

544 creating new tables as necessary. 

545 

546 Parameters 

547 ---------- 

548 element : `DimensionElement` 

549 Element for which a table should created (as necessary) and 

550 an associated `DimensionRecordStorage` returned. 

551 

552 Returns 

553 ------- 

554 records : `DimensionRecordStorage` 

555 The object representing the records for the given element in this 

556 layer. 

557 

558 Raises 

559 ------ 

560 TransactionInterruption 

561 Raised if this operation is invoked within a `Database.transaction` 

562 context. 

563 """ 

564 raise NotImplementedError() 

565 

566 @abstractmethod 

567 def saveDimensionGraph(self, graph: DimensionGraph) -> int: 

568 """Save a `DimensionGraph` definition to the database, allowing it to 

569 be retrieved later via the returned key. 

570 

571 Parameters 

572 ---------- 

573 graph : `DimensionGraph` 

574 Set of dimensions to save. 

575 

576 Returns 

577 ------- 

578 key : `int` 

579 Integer used as the unique key for this `DimensionGraph` in the 

580 database. 

581 

582 Raises 

583 ------ 

584 TransactionInterruption 

585 Raised if this operation is invoked within a `Database.transaction` 

586 context. 

587 """ 

588 raise NotImplementedError() 

589 

590 @abstractmethod 

591 def loadDimensionGraph(self, key: int) -> DimensionGraph: 

592 """Retrieve a `DimensionGraph` that was previously saved in the 

593 database. 

594 

595 Parameters 

596 ---------- 

597 key : `int` 

598 Integer used as the unique key for this `DimensionGraph` in the 

599 database. 

600 

601 Returns 

602 ------- 

603 graph : `DimensionGraph` 

604 Retrieved graph. 

605 

606 Raises 

607 ------ 

608 KeyError 

609 Raised if the given key cannot be found in the database. 

610 """ 

611 raise NotImplementedError() 

612 

613 @abstractmethod 

614 def clearCaches(self) -> None: 

615 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

616 instances. 

617 

618 This is called by `Registry` when transactions are rolled back, to 

619 avoid in-memory caches from ever containing records that are not 

620 present in persistent storage. 

621 """ 

622 raise NotImplementedError() 

623 

624 universe: DimensionUniverse 

625 """Universe of all dimensions and dimension elements known to the 

626 `Registry` (`DimensionUniverse`). 

627 """