Coverage for python/lsst/daf/butler/registry/interfaces/_dimensions.py: 69%

118 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-24 23:50 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "DatabaseDimensionOverlapStorage", 

25 "DatabaseDimensionRecordStorage", 

26 "DimensionRecordStorage", 

27 "DimensionRecordStorageManager", 

28 "GovernorDimensionRecordStorage", 

29 "SkyPixDimensionRecordStorage", 

30) 

31 

32from abc import ABC, abstractmethod 

33from typing import TYPE_CHECKING, AbstractSet, Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union 

34 

35import sqlalchemy 

36 

37from ...core import DatabaseDimensionElement, DimensionGraph, GovernorDimension, SkyPixDimension 

38from ._versioning import VersionedExtension 

39 

40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 from ...core import ( 

42 DataCoordinateIterable, 

43 DimensionElement, 

44 DimensionRecord, 

45 DimensionUniverse, 

46 NamedKeyDict, 

47 NamedKeyMapping, 

48 TimespanDatabaseRepresentation, 

49 ) 

50 from ..queries import QueryBuilder 

51 from ._database import Database, StaticTablesContext 

52 

53 

54OverlapSide = Union[SkyPixDimension, Tuple[DatabaseDimensionElement, str]] 

55 

56 

57class DimensionRecordStorage(ABC): 

58 """An abstract base class that represents a way of storing the records 

59 associated with a single `DimensionElement`. 

60 

61 Concrete `DimensionRecordStorage` instances should generally be constructed 

62 via a call to `setupDimensionStorage`, which selects the appropriate 

63 subclass for each element according to its configuration. 

64 

65 All `DimensionRecordStorage` methods are pure abstract, even though in some 

66 cases a reasonable default implementation might be possible, in order to 

67 better guarantee all methods are correctly overridden. All of these 

68 potentially-defaultable implementations are extremely trivial, so asking 

69 subclasses to provide them is not a significant burden. 

70 """ 

71 

72 @property 

73 @abstractmethod 

74 def element(self) -> DimensionElement: 

75 """The element whose records this instance managers 

76 (`DimensionElement`). 

77 """ 

78 raise NotImplementedError() 

79 

80 @abstractmethod 

81 def clearCaches(self) -> None: 

82 """Clear any in-memory caches held by the storage instance. 

83 

84 This is called by `Registry` when transactions are rolled back, to 

85 avoid in-memory caches from ever containing records that are not 

86 present in persistent storage. 

87 """ 

88 raise NotImplementedError() 

89 

90 @abstractmethod 

91 def join( 

92 self, 

93 builder: QueryBuilder, 

94 *, 

95 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

96 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None, 

97 ) -> sqlalchemy.sql.FromClause: 

98 """Add the dimension element's logical table to a query under 

99 construction. 

100 

101 This is a visitor pattern interface that is expected to be called only 

102 by `QueryBuilder.joinDimensionElement`. 

103 

104 Parameters 

105 ---------- 

106 builder : `QueryBuilder` 

107 Builder for the query that should contain this element. 

108 regions : `NamedKeyDict`, optional 

109 A mapping from `DimensionElement` to a SQLAlchemy column containing 

110 the region for that element, which should be updated to include a 

111 region column for this element if one exists. If `None`, 

112 ``self.element`` is not being included in the query via a spatial 

113 join. 

114 timespan : `NamedKeyDict`, optional 

115 A mapping from `DimensionElement` to a `Timespan` of SQLALchemy 

116 columns containing the timespan for that element, which should be 

117 updated to include timespan columns for this element if they exist. 

118 If `None`, ``self.element`` is not being included in the query via 

119 a temporal join. 

120 

121 Returns 

122 ------- 

123 fromClause : `sqlalchemy.sql.FromClause` 

124 Table or clause for the element which is joined. 

125 

126 Notes 

127 ----- 

128 Elements are only included in queries via spatial and/or temporal joins 

129 when necessary to connect them to other elements in the query, so 

130 ``regions`` and ``timespans`` cannot be assumed to be not `None` just 

131 because an element has a region or timespan. 

132 """ 

133 raise NotImplementedError() 

134 

135 @abstractmethod 

136 def insert(self, *records: DimensionRecord, replace: bool = False, skip_existing: bool = False) -> None: 

137 """Insert one or more records into storage. 

138 

139 Parameters 

140 ---------- 

141 records 

142 One or more instances of the `DimensionRecord` subclass for the 

143 element this storage is associated with. 

144 replace: `bool`, optional 

145 If `True` (`False` is default), replace existing records in the 

146 database if there is a conflict. 

147 skip_existing : `bool`, optional 

148 If `True` (`False` is default), skip insertion if a record with 

149 the same primary key values already exists. 

150 

151 Raises 

152 ------ 

153 TypeError 

154 Raised if the element does not support record insertion. 

155 sqlalchemy.exc.IntegrityError 

156 Raised if one or more records violate database integrity 

157 constraints. 

158 

159 Notes 

160 ----- 

161 As `insert` is expected to be called only by a `Registry`, we rely 

162 on `Registry` to provide transactionality, both by using a SQLALchemy 

163 connection shared with the `Registry` and by relying on it to call 

164 `clearCaches` when rolling back transactions. 

165 """ 

166 raise NotImplementedError() 

167 

168 @abstractmethod 

169 def sync(self, record: DimensionRecord, update: bool = False) -> Union[bool, Dict[str, Any]]: 

170 """Synchronize a record with the database, inserting it only if it does 

171 not exist and comparing values if it does. 

172 

173 Parameters 

174 ---------- 

175 record : `DimensionRecord`. 

176 An instance of the `DimensionRecord` subclass for the 

177 element this storage is associated with. 

178 update: `bool`, optional 

179 If `True` (`False` is default), update the existing record in the 

180 database if there is a conflict. 

181 

182 Returns 

183 ------- 

184 inserted_or_updated : `bool` or `dict` 

185 `True` if a new row was inserted, `False` if no changes were 

186 needed, or a `dict` mapping updated column names to their old 

187 values if an update was performed (only possible if 

188 ``update=True``). 

189 

190 Raises 

191 ------ 

192 DatabaseConflictError 

193 Raised if the record exists in the database (according to primary 

194 key lookup) but is inconsistent with the given one. 

195 TypeError 

196 Raised if the element does not support record synchronization. 

197 sqlalchemy.exc.IntegrityError 

198 Raised if one or more records violate database integrity 

199 constraints. 

200 """ 

201 raise NotImplementedError() 

202 

203 @abstractmethod 

204 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

205 """Retrieve records from storage. 

206 

207 Parameters 

208 ---------- 

209 dataIds : `DataCoordinateIterable` 

210 Data IDs that identify the records to be retrieved. 

211 

212 Returns 

213 ------- 

214 records : `Iterable` [ `DimensionRecord` ] 

215 Record retrieved from storage. Not all data IDs may have 

216 corresponding records (if there are no records that match a data 

217 ID), and even if they are, the order of inputs is not preserved. 

218 """ 

219 raise NotImplementedError() 

220 

221 @abstractmethod 

222 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

223 """Return tables used for schema digest. 

224 

225 Returns 

226 ------- 

227 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

228 Possibly empty set of tables for schema digest calculations. 

229 """ 

230 raise NotImplementedError() 

231 

232 

233class GovernorDimensionRecordStorage(DimensionRecordStorage): 

234 """Intermediate interface for `DimensionRecordStorage` objects that provide 

235 storage for `GovernorDimension` instances. 

236 """ 

237 

238 @classmethod 

239 @abstractmethod 

240 def initialize( 

241 cls, 

242 db: Database, 

243 dimension: GovernorDimension, 

244 *, 

245 context: Optional[StaticTablesContext] = None, 

246 config: Mapping[str, Any], 

247 ) -> GovernorDimensionRecordStorage: 

248 """Construct an instance of this class using a standardized interface. 

249 

250 Parameters 

251 ---------- 

252 db : `Database` 

253 Interface to the underlying database engine and namespace. 

254 dimension : `GovernorDimension` 

255 Dimension the new instance will manage records for. 

256 context : `StaticTablesContext`, optional 

257 If provided, an object to use to create any new tables. If not 

258 provided, ``db.ensureTableExists`` should be used instead. 

259 config : `Mapping` 

260 Extra configuration options specific to the implementation. 

261 

262 Returns 

263 ------- 

264 storage : `GovernorDimensionRecordStorage` 

265 A new `GovernorDimensionRecordStorage` subclass instance. 

266 """ 

267 raise NotImplementedError() 

268 

269 @property 

270 @abstractmethod 

271 def element(self) -> GovernorDimension: 

272 # Docstring inherited from DimensionRecordStorage. 

273 raise NotImplementedError() 

274 

275 @abstractmethod 

276 def refresh(self) -> None: 

277 """Ensure all other operations on this manager are aware of any 

278 changes made by other clients since it was initialized or last 

279 refreshed. 

280 """ 

281 raise NotImplementedError() 

282 

283 @property 

284 @abstractmethod 

285 def values(self) -> AbstractSet[str]: 

286 """All primary key values for this dimension (`set` [ `str` ]). 

287 

288 This may rely on an in-memory cache and hence not reflect changes to 

289 the set of values made by other `Butler` / `Registry` clients. Call 

290 `refresh` to ensure up-to-date results. 

291 """ 

292 raise NotImplementedError() 

293 

294 @property 

295 @abstractmethod 

296 def table(self) -> sqlalchemy.schema.Table: 

297 """The SQLAlchemy table that backs this dimension 

298 (`sqlalchemy.schema.Table`). 

299 """ 

300 raise NotImplementedError 

301 

302 @abstractmethod 

303 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None: 

304 """Add a function or method to be called after new records for this 

305 dimension are inserted by `insert` or `sync`. 

306 

307 Parameters 

308 ---------- 

309 callback 

310 Callable that takes a single `DimensionRecord` argument. This will 

311 be called immediately after any successful insertion, in the same 

312 transaction. 

313 """ 

314 raise NotImplementedError() 

315 

316 

317class SkyPixDimensionRecordStorage(DimensionRecordStorage): 

318 """Intermediate interface for `DimensionRecordStorage` objects that provide 

319 storage for `SkyPixDimension` instances. 

320 """ 

321 

322 @property 

323 @abstractmethod 

324 def element(self) -> SkyPixDimension: 

325 # Docstring inherited from DimensionRecordStorage. 

326 raise NotImplementedError() 

327 

328 

329class DatabaseDimensionRecordStorage(DimensionRecordStorage): 

330 """Intermediate interface for `DimensionRecordStorage` objects that provide 

331 storage for `DatabaseDimensionElement` instances. 

332 """ 

333 

334 @classmethod 

335 @abstractmethod 

336 def initialize( 

337 cls, 

338 db: Database, 

339 element: DatabaseDimensionElement, 

340 *, 

341 context: Optional[StaticTablesContext] = None, 

342 config: Mapping[str, Any], 

343 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

344 ) -> DatabaseDimensionRecordStorage: 

345 """Construct an instance of this class using a standardized interface. 

346 

347 Parameters 

348 ---------- 

349 db : `Database` 

350 Interface to the underlying database engine and namespace. 

351 element : `DatabaseDimensionElement` 

352 Dimension element the new instance will manage records for. 

353 context : `StaticTablesContext`, optional 

354 If provided, an object to use to create any new tables. If not 

355 provided, ``db.ensureTableExists`` should be used instead. 

356 config : `Mapping` 

357 Extra configuration options specific to the implementation. 

358 governors : `NamedKeyMapping` 

359 Mapping containing all governor dimension storage implementations. 

360 

361 Returns 

362 ------- 

363 storage : `DatabaseDimensionRecordStorage` 

364 A new `DatabaseDimensionRecordStorage` subclass instance. 

365 """ 

366 raise NotImplementedError() 

367 

368 @property 

369 @abstractmethod 

370 def element(self) -> DatabaseDimensionElement: 

371 # Docstring inherited from DimensionRecordStorage. 

372 raise NotImplementedError() 

373 

374 def connect(self, overlaps: DatabaseDimensionOverlapStorage) -> None: 

375 """Inform this record storage object of the object that will manage 

376 the overlaps between this element and another element. 

377 

378 This will only be called if ``self.element.spatial is not None``, 

379 and will be called immediately after construction (before any other 

380 methods). In the future, implementations will be required to call a 

381 method on any connected overlap storage objects any time new records 

382 for the element are inserted. 

383 

384 Parameters 

385 ---------- 

386 overlaps : `DatabaseDimensionRecordStorage` 

387 Object managing overlaps between this element and another 

388 database-backed element. 

389 """ 

390 raise NotImplementedError(f"{type(self).__name__} does not support spatial elements.") 

391 

392 

393class DatabaseDimensionOverlapStorage(ABC): 

394 """A base class for objects that manage overlaps between a pair of 

395 database-backed dimensions. 

396 """ 

397 

398 @classmethod 

399 @abstractmethod 

400 def initialize( 

401 cls, 

402 db: Database, 

403 elementStorage: Tuple[DatabaseDimensionRecordStorage, DatabaseDimensionRecordStorage], 

404 governorStorage: Tuple[GovernorDimensionRecordStorage, GovernorDimensionRecordStorage], 

405 context: Optional[StaticTablesContext] = None, 

406 ) -> DatabaseDimensionOverlapStorage: 

407 """Construct an instance of this class using a standardized interface. 

408 

409 Parameters 

410 ---------- 

411 db : `Database` 

412 Interface to the underlying database engine and namespace. 

413 elementStorage : `tuple` [ `DatabaseDimensionRecordStorage` ] 

414 Storage objects for the elements this object will related. 

415 governorStorage : `tuple` [ `GovernorDimensionRecordStorage` ] 

416 Storage objects for the governor dimensions of the elements this 

417 object will related. 

418 context : `StaticTablesContext`, optional 

419 If provided, an object to use to create any new tables. If not 

420 provided, ``db.ensureTableExists`` should be used instead. 

421 

422 Returns 

423 ------- 

424 storage : `DatabaseDimensionOverlapStorage` 

425 A new `DatabaseDimensionOverlapStorage` subclass instance. 

426 """ 

427 raise NotImplementedError() 

428 

429 @property 

430 @abstractmethod 

431 def elements(self) -> Tuple[DatabaseDimensionElement, DatabaseDimensionElement]: 

432 """The pair of elements whose overlaps this object manages. 

433 

434 The order of elements is the same as their ordering within the 

435 `DimensionUniverse`. 

436 """ 

437 raise NotImplementedError() 

438 

439 @abstractmethod 

440 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

441 """Return tables used for schema digest. 

442 

443 Returns 

444 ------- 

445 tables : `Iterable` [ `sqlalchemy.schema.Table` ] 

446 Possibly empty set of tables for schema digest calculations. 

447 """ 

448 raise NotImplementedError() 

449 

450 

451class DimensionRecordStorageManager(VersionedExtension): 

452 """An interface for managing the dimension records in a `Registry`. 

453 

454 `DimensionRecordStorageManager` primarily serves as a container and factory 

455 for `DimensionRecordStorage` instances, which each provide access to the 

456 records for a different `DimensionElement`. 

457 

458 Parameters 

459 ---------- 

460 universe : `DimensionUniverse` 

461 Universe of all dimensions and dimension elements known to the 

462 `Registry`. 

463 

464 Notes 

465 ----- 

466 In a multi-layer `Registry`, many dimension elements will only have 

467 records in one layer (often the base layer). The union of the records 

468 across all layers forms the logical table for the full `Registry`. 

469 """ 

470 

471 def __init__(self, *, universe: DimensionUniverse): 

472 self.universe = universe 

473 

474 @classmethod 

475 @abstractmethod 

476 def initialize( 

477 cls, db: Database, context: StaticTablesContext, *, universe: DimensionUniverse 

478 ) -> DimensionRecordStorageManager: 

479 """Construct an instance of the manager. 

480 

481 Parameters 

482 ---------- 

483 db : `Database` 

484 Interface to the underlying database engine and namespace. 

485 context : `StaticTablesContext` 

486 Context object obtained from `Database.declareStaticTables`; used 

487 to declare any tables that should always be present in a layer 

488 implemented with this manager. 

489 universe : `DimensionUniverse` 

490 Universe graph containing dimensions known to this `Registry`. 

491 

492 Returns 

493 ------- 

494 manager : `DimensionRecordStorageManager` 

495 An instance of a concrete `DimensionRecordStorageManager` subclass. 

496 """ 

497 raise NotImplementedError() 

498 

499 @abstractmethod 

500 def refresh(self) -> None: 

501 """Ensure all other operations on this manager are aware of any 

502 changes made by other clients since it was initialized or last 

503 refreshed. 

504 """ 

505 raise NotImplementedError() 

506 

507 def __getitem__(self, element: DimensionElement) -> DimensionRecordStorage: 

508 """Interface to `get` that raises `LookupError` instead of returning 

509 `None` on failure. 

510 """ 

511 r = self.get(element) 

512 if r is None: 

513 raise LookupError(f"No dimension element '{element.name}' found in this registry layer.") 

514 return r 

515 

516 @abstractmethod 

517 def get(self, element: DimensionElement) -> Optional[DimensionRecordStorage]: 

518 """Return an object that provides access to the records associated with 

519 the given element, if one exists in this layer. 

520 

521 Parameters 

522 ---------- 

523 element : `DimensionElement` 

524 Element for which records should be returned. 

525 

526 Returns 

527 ------- 

528 records : `DimensionRecordStorage` or `None` 

529 The object representing the records for the given element in this 

530 layer, or `None` if there are no records for that element in this 

531 layer. 

532 

533 Notes 

534 ----- 

535 Dimension elements registered by another client of the same layer since 

536 the last call to `initialize` or `refresh` may not be found. 

537 """ 

538 raise NotImplementedError() 

539 

540 @abstractmethod 

541 def register(self, element: DimensionElement) -> DimensionRecordStorage: 

542 """Ensure that this layer can hold records for the given element, 

543 creating new tables as necessary. 

544 

545 Parameters 

546 ---------- 

547 element : `DimensionElement` 

548 Element for which a table should created (as necessary) and 

549 an associated `DimensionRecordStorage` returned. 

550 

551 Returns 

552 ------- 

553 records : `DimensionRecordStorage` 

554 The object representing the records for the given element in this 

555 layer. 

556 

557 Raises 

558 ------ 

559 TransactionInterruption 

560 Raised if this operation is invoked within a `Database.transaction` 

561 context. 

562 """ 

563 raise NotImplementedError() 

564 

565 @abstractmethod 

566 def saveDimensionGraph(self, graph: DimensionGraph) -> int: 

567 """Save a `DimensionGraph` definition to the database, allowing it to 

568 be retrieved later via the returned key. 

569 

570 Parameters 

571 ---------- 

572 graph : `DimensionGraph` 

573 Set of dimensions to save. 

574 

575 Returns 

576 ------- 

577 key : `int` 

578 Integer used as the unique key for this `DimensionGraph` in the 

579 database. 

580 

581 Raises 

582 ------ 

583 TransactionInterruption 

584 Raised if this operation is invoked within a `Database.transaction` 

585 context. 

586 """ 

587 raise NotImplementedError() 

588 

589 @abstractmethod 

590 def loadDimensionGraph(self, key: int) -> DimensionGraph: 

591 """Retrieve a `DimensionGraph` that was previously saved in the 

592 database. 

593 

594 Parameters 

595 ---------- 

596 key : `int` 

597 Integer used as the unique key for this `DimensionGraph` in the 

598 database. 

599 

600 Returns 

601 ------- 

602 graph : `DimensionGraph` 

603 Retrieved graph. 

604 

605 Raises 

606 ------ 

607 KeyError 

608 Raised if the given key cannot be found in the database. 

609 """ 

610 raise NotImplementedError() 

611 

612 @abstractmethod 

613 def clearCaches(self) -> None: 

614 """Clear any in-memory caches held by nested `DimensionRecordStorage` 

615 instances. 

616 

617 This is called by `Registry` when transactions are rolled back, to 

618 avoid in-memory caches from ever containing records that are not 

619 present in persistent storage. 

620 """ 

621 raise NotImplementedError() 

622 

623 universe: DimensionUniverse 

624 """Universe of all dimensions and dimension elements known to the 

625 `Registry` (`DimensionUniverse`). 

626 """